diff --git a/README.md b/README.md index cb3d56de..e4f5dd18 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,16 @@ --- -layout: default +layout: forward +target: https://developers.google.com/mediapipe title: Home nav_order: 1 --- ---- -**Attention:** *Thanks for your interest in MediaPipe! We have moved to +**Attention:** *We have moved to [https://developers.google.com/mediapipe](https://developers.google.com/mediapipe) as the primary developer documentation site for MediaPipe as of April 3, 2023.* -*This notice and web page will be removed on June 1, 2023.* - ![MediaPipe](https://developers.google.com/static/mediapipe/images/home/hero_01_1920.png) **Attention**: MediaPipe Solutions Preview is an early release. [Learn diff --git a/WORKSPACE b/WORKSPACE index 7c5841f7..9b50ff8e 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -266,10 +266,10 @@ http_archive( http_archive( name = "com_googlesource_code_re2", - sha256 = "e06b718c129f4019d6e7aa8b7631bee38d3d450dd980246bfaf493eb7db67868", - strip_prefix = "re2-fe4a310131c37f9a7e7f7816fa6ce2a8b27d65a8", + sha256 = "ef516fb84824a597c4d5d0d6d330daedb18363b5a99eda87d027e6bdd9cba299", + strip_prefix = "re2-03da4fc0857c285e3a26782f6bc8931c4c950df4", urls = [ - "https://github.com/google/re2/archive/fe4a310131c37f9a7e7f7816fa6ce2a8b27d65a8.tar.gz", + "https://github.com/google/re2/archive/03da4fc0857c285e3a26782f6bc8931c4c950df4.tar.gz", ], ) diff --git a/docs/index.md b/docs/index.md index cb3d56de..e4f5dd18 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,17 +1,16 @@ --- -layout: default +layout: forward +target: https://developers.google.com/mediapipe title: Home nav_order: 1 --- ---- -**Attention:** *Thanks for your interest in MediaPipe! We have moved to +**Attention:** *We have moved to [https://developers.google.com/mediapipe](https://developers.google.com/mediapipe) as the primary developer documentation site for MediaPipe as of April 3, 2023.* -*This notice and web page will be removed on June 1, 2023.* - ![MediaPipe](https://developers.google.com/static/mediapipe/images/home/hero_01_1920.png) **Attention**: MediaPipe Solutions Preview is an early release. [Learn diff --git a/docs/solutions/face_detection.md b/docs/solutions/face_detection.md index f060d062..93f239c3 100644 --- a/docs/solutions/face_detection.md +++ b/docs/solutions/face_detection.md @@ -20,9 +20,9 @@ nav_order: 1 --- **Attention:** *Thank you for your interest in MediaPipe Solutions. -As of March 1, 2023, this solution is planned to be upgraded to a new MediaPipe +As of May 10, 2023, this solution was upgraded to a new MediaPipe Solution. For more information, see the -[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy) +[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/vision/face_detector) site.* ---- diff --git a/docs/solutions/face_mesh.md b/docs/solutions/face_mesh.md index ab34ba40..a859bafa 100644 --- a/docs/solutions/face_mesh.md +++ b/docs/solutions/face_mesh.md @@ -20,9 +20,9 @@ nav_order: 2 --- **Attention:** *Thank you for your interest in MediaPipe Solutions. -As of March 1, 2023, this solution is planned to be upgraded to a new MediaPipe +As of May 10, 2023, this solution was upgraded to a new MediaPipe Solution. For more information, see the -[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy) +[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/vision/face_landmarker) site.* ---- diff --git a/docs/solutions/iris.md b/docs/solutions/iris.md index eab3dedf..c0af4342 100644 --- a/docs/solutions/iris.md +++ b/docs/solutions/iris.md @@ -20,9 +20,9 @@ nav_order: 3 --- **Attention:** *Thank you for your interest in MediaPipe Solutions. -As of March 1, 2023, this solution is planned to be upgraded to a new MediaPipe +As of May 10, 2023, this solution was upgraded to a new MediaPipe Solution. For more information, see the -[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy) +[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/vision/face_landmarker) site.* ---- diff --git a/docs/solutions/pose.md b/docs/solutions/pose.md index b6f9408e..09c313b5 100644 --- a/docs/solutions/pose.md +++ b/docs/solutions/pose.md @@ -22,9 +22,9 @@ nav_order: 5 --- **Attention:** *Thank you for your interest in MediaPipe Solutions. -As of March 1, 2023, this solution is planned to be upgraded to a new MediaPipe +As of May 10, 2023, this solution was upgraded to a new MediaPipe Solution. For more information, see the -[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/vision/pose_landmarker/) +[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/vision/pose_landmarker) site.* ---- diff --git a/docs/solutions/pose_classification.md b/docs/solutions/pose_classification.md index 8420e2d7..091b0c99 100644 --- a/docs/solutions/pose_classification.md +++ b/docs/solutions/pose_classification.md @@ -21,7 +21,7 @@ nav_order: 1 --- **Attention:** *Thank you for your interest in MediaPipe Solutions. -As of March 1, 2023, this solution is planned to be upgraded to a new MediaPipe +As of May 10, 2023, this solution was upgraded to a new MediaPipe Solution. For more information, see the [MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/vision/pose_landmarker/) site.* diff --git a/docs/solutions/solutions.md b/docs/solutions/solutions.md index 7bc32d16..10551b7c 100644 --- a/docs/solutions/solutions.md +++ b/docs/solutions/solutions.md @@ -1,5 +1,6 @@ --- -layout: default +layout: forward +target: https://developers.google.com/mediapipe/solutions/guide#legacy title: MediaPipe Legacy Solutions nav_order: 3 has_children: true @@ -13,8 +14,7 @@ has_toc: false {:toc} --- -**Attention:** *Thank you for your interest in MediaPipe Solutions. We have -ended support for +**Attention:** *We have ended support for [these MediaPipe Legacy Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy) as of March 1, 2023. All other [MediaPipe Legacy Solutions will be upgraded](https://developers.google.com/mediapipe/solutions/guide#legacy) @@ -25,14 +25,6 @@ be provided on an as-is basis. We encourage you to check out the new MediaPipe Solutions at: [https://developers.google.com/mediapipe/solutions](https://developers.google.com/mediapipe/solutions)* -*This notice and web page will be removed on June 1, 2023.* - ----- - -









-









-









- ---- MediaPipe offers open source cross-platform, customizable ML solutions for live diff --git a/mediapipe/calculators/core/BUILD b/mediapipe/calculators/core/BUILD index 297a416d..d773bef3 100644 --- a/mediapipe/calculators/core/BUILD +++ b/mediapipe/calculators/core/BUILD @@ -1240,6 +1240,7 @@ cc_library( "//mediapipe/framework/formats:classification_cc_proto", "//mediapipe/framework/formats:detection_cc_proto", "//mediapipe/framework/formats:landmark_cc_proto", + "//mediapipe/framework/formats:rect_cc_proto", "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", ], diff --git a/mediapipe/calculators/core/get_vector_item_calculator.cc b/mediapipe/calculators/core/get_vector_item_calculator.cc index 3306e4ff..5dbda6d9 100644 --- a/mediapipe/calculators/core/get_vector_item_calculator.cc +++ b/mediapipe/calculators/core/get_vector_item_calculator.cc @@ -17,6 +17,7 @@ #include "mediapipe/framework/formats/classification.pb.h" #include "mediapipe/framework/formats/detection.pb.h" #include "mediapipe/framework/formats/landmark.pb.h" +#include "mediapipe/framework/formats/rect.pb.h" namespace mediapipe { namespace api2 { @@ -37,5 +38,12 @@ using GetDetectionVectorItemCalculator = GetVectorItemCalculator; REGISTER_CALCULATOR(GetDetectionVectorItemCalculator); +using GetNormalizedRectVectorItemCalculator = + GetVectorItemCalculator; +REGISTER_CALCULATOR(GetNormalizedRectVectorItemCalculator); + +using GetRectVectorItemCalculator = GetVectorItemCalculator; +REGISTER_CALCULATOR(GetRectVectorItemCalculator); + } // namespace api2 } // namespace mediapipe diff --git a/mediapipe/calculators/util/landmarks_smoothing_calculator_utils.cc b/mediapipe/calculators/util/landmarks_smoothing_calculator_utils.cc index a381c823..32e28215 100644 --- a/mediapipe/calculators/util/landmarks_smoothing_calculator_utils.cc +++ b/mediapipe/calculators/util/landmarks_smoothing_calculator_utils.cc @@ -290,17 +290,17 @@ void LandmarksToNormalizedLandmarks(const LandmarkList& landmarks, // Scale Z the same way as X (using image width). norm_landmark->set_z(landmark.z() / image_width); - if (landmark.has_presence()) { - norm_landmark->set_presence(landmark.presence()); - } else { - norm_landmark->clear_presence(); - } - if (landmark.has_visibility()) { norm_landmark->set_visibility(landmark.visibility()); } else { norm_landmark->clear_visibility(); } + + if (landmark.has_presence()) { + norm_landmark->set_presence(landmark.presence()); + } else { + norm_landmark->clear_presence(); + } } } diff --git a/mediapipe/framework/api2/packet.h b/mediapipe/framework/api2/packet.h index b1ebb041..c059a988 100644 --- a/mediapipe/framework/api2/packet.h +++ b/mediapipe/framework/api2/packet.h @@ -165,7 +165,7 @@ template struct IsCompatibleType> : std::integral_constant || ...)> {}; -}; // namespace internal +} // namespace internal template inline Packet PacketBase::As() const { @@ -259,19 +259,19 @@ struct First { template struct AddStatus { - using type = StatusOr; + using type = absl::StatusOr; }; template -struct AddStatus> { - using type = StatusOr; +struct AddStatus> { + using type = absl::StatusOr; }; template <> -struct AddStatus { - using type = Status; +struct AddStatus { + using type = absl::Status; }; template <> struct AddStatus { - using type = Status; + using type = absl::Status; }; template @@ -282,7 +282,7 @@ struct CallAndAddStatusImpl { }; template struct CallAndAddStatusImpl { - Status operator()(const F& f, A&&... a) { + absl::Status operator()(const F& f, A&&... a) { f(std::forward(a)...); return {}; } diff --git a/mediapipe/framework/deps/safe_int.h b/mediapipe/framework/deps/safe_int.h index 4c120bc1..f6dbb931 100644 --- a/mediapipe/framework/deps/safe_int.h +++ b/mediapipe/framework/deps/safe_int.h @@ -88,10 +88,13 @@ class SafeIntStrongIntValidator { // If the argument is floating point, we can do a simple check to make // sure the value is in range. It is undefined behavior to convert to int - // from a float that is out of range. + // from a float that is out of range. Since large integers will loose some + // precision when being converted to floating point, the integer max and min + // are explicitly converted back to floating point for this comparison, in + // order to satisfy compiler warnings. if (std::is_floating_point::value) { - if (arg < std::numeric_limits::min() || - arg > std::numeric_limits::max()) { + if (arg < static_cast(std::numeric_limits::min()) || + arg > static_cast(std::numeric_limits::max())) { ErrorType::Error("SafeInt: init from out of bounds float", arg, "="); } } else { @@ -284,11 +287,11 @@ class SafeIntStrongIntValidator { // A SafeIntStrongIntValidator policy class to LOG(FATAL) on errors. struct LogFatalOnError { template - static void Error(const char *error, Tlhs lhs, Trhs rhs, const char *op) { + static void Error(const char* error, Tlhs lhs, Trhs rhs, const char* op) { LOG(FATAL) << error << ": (" << lhs << " " << op << " " << rhs << ")"; } template - static void Error(const char *error, Tval val, const char *op) { + static void Error(const char* error, Tval val, const char* op) { LOG(FATAL) << error << ": (" << op << val << ")"; } }; diff --git a/mediapipe/framework/output_stream_shard.cc b/mediapipe/framework/output_stream_shard.cc index 1b096efb..682c704c 100644 --- a/mediapipe/framework/output_stream_shard.cc +++ b/mediapipe/framework/output_stream_shard.cc @@ -94,7 +94,7 @@ const Packet& OutputStreamShard::Header() const { // binary. This function can be defined in the .cc file because only two // versions are ever instantiated, and all call sites are within this .cc file. template -Status OutputStreamShard::AddPacketInternal(T&& packet) { +absl::Status OutputStreamShard::AddPacketInternal(T&& packet) { if (IsClosed()) { return mediapipe::FailedPreconditionErrorBuilder(MEDIAPIPE_LOC) << "Packet sent to closed stream \"" << Name() << "\"."; @@ -113,7 +113,7 @@ Status OutputStreamShard::AddPacketInternal(T&& packet) { << timestamp.DebugString(); } - Status result = output_stream_spec_->packet_type->Validate(packet); + absl::Status result = output_stream_spec_->packet_type->Validate(packet); if (!result.ok()) { return StatusBuilder(result, MEDIAPIPE_LOC).SetPrepend() << absl::StrCat( "Packet type mismatch on calculator outputting to stream \"", @@ -132,14 +132,14 @@ Status OutputStreamShard::AddPacketInternal(T&& packet) { } void OutputStreamShard::AddPacket(const Packet& packet) { - Status status = AddPacketInternal(packet); + absl::Status status = AddPacketInternal(packet); if (!status.ok()) { output_stream_spec_->TriggerErrorCallback(status); } } void OutputStreamShard::AddPacket(Packet&& packet) { - Status status = AddPacketInternal(std::move(packet)); + absl::Status status = AddPacketInternal(std::move(packet)); if (!status.ok()) { output_stream_spec_->TriggerErrorCallback(status); } diff --git a/mediapipe/framework/tool/status_util.cc b/mediapipe/framework/tool/status_util.cc index 401a1b63..0c277a00 100644 --- a/mediapipe/framework/tool/status_util.cc +++ b/mediapipe/framework/tool/status_util.cc @@ -59,8 +59,8 @@ absl::Status CombinedStatus(absl::string_view general_comment, } } } - if (error_code == StatusCode::kOk) return OkStatus(); - Status combined; + if (error_code == absl::StatusCode::kOk) return absl::OkStatus(); + absl::Status combined; combined = absl::Status( error_code, absl::StrCat(general_comment, "\n", absl::StrJoin(errors, "\n"))); diff --git a/mediapipe/framework/type_map.h b/mediapipe/framework/type_map.h index 8fb324e9..42f6fe6b 100644 --- a/mediapipe/framework/type_map.h +++ b/mediapipe/framework/type_map.h @@ -241,9 +241,9 @@ class StaticMap { #define DEFINE_MEDIAPIPE_TYPE_MAP(MapName, KeyType) \ class MapName : public type_map_internal::StaticMap {}; // Defines a map from unique typeid number to MediaPipeTypeData. -DEFINE_MEDIAPIPE_TYPE_MAP(PacketTypeIdToMediaPipeTypeData, size_t); +DEFINE_MEDIAPIPE_TYPE_MAP(PacketTypeIdToMediaPipeTypeData, size_t) // Defines a map from unique type string to MediaPipeTypeData. -DEFINE_MEDIAPIPE_TYPE_MAP(PacketTypeStringToMediaPipeTypeData, std::string); +DEFINE_MEDIAPIPE_TYPE_MAP(PacketTypeStringToMediaPipeTypeData, std::string) // MEDIAPIPE_REGISTER_TYPE can be used to register a type. // Convention: diff --git a/mediapipe/gpu/gl_calculator_helper.cc b/mediapipe/gpu/gl_calculator_helper.cc index 9b217ddf..974525a9 100644 --- a/mediapipe/gpu/gl_calculator_helper.cc +++ b/mediapipe/gpu/gl_calculator_helper.cc @@ -211,6 +211,14 @@ GlTexture GlCalculatorHelper::CreateDestinationTexture(int width, int height, return MapGpuBuffer(gpu_buffer, gpu_buffer.GetWriteView(0)); } +GlTexture GlCalculatorHelper::CreateDestinationTexture( + const ImageFrame& image_frame) { + // TODO: ensure buffer pool is used when creating textures out of + // ImageFrame. + GpuBuffer gpu_buffer = GpuBufferCopyingImageFrame(image_frame); + return MapGpuBuffer(gpu_buffer, gpu_buffer.GetWriteView(0)); +} + GlTexture GlCalculatorHelper::CreateSourceTexture( const mediapipe::Image& image) { return CreateSourceTexture(image.GetGpuBuffer()); diff --git a/mediapipe/gpu/gl_calculator_helper.h b/mediapipe/gpu/gl_calculator_helper.h index af897bbe..c1b94fa8 100644 --- a/mediapipe/gpu/gl_calculator_helper.h +++ b/mediapipe/gpu/gl_calculator_helper.h @@ -135,6 +135,12 @@ class GlCalculatorHelper { // This is deprecated because: 1) it encourages the use of GlTexture as a // long-lived object; 2) it requires copying the ImageFrame's contents, // which may not always be necessary. + // + // WARNING: do NOT use as a destination texture which will be sent to + // downstream calculators as it may lead to synchronization issues. The result + // is meant to be a short-lived object, local to a single calculator and + // single GL thread. Use `CreateDestinationTexture` instead, if you need a + // destination texture. ABSL_DEPRECATED("Use `GpuBufferWithImageFrame`.") GlTexture CreateSourceTexture(const ImageFrame& image_frame); @@ -156,6 +162,14 @@ class GlCalculatorHelper { int output_width, int output_height, GpuBufferFormat format = GpuBufferFormat::kBGRA32); + // Creates a destination texture copying and uploading passed image frame. + // + // WARNING: mind that this functions creates a new texture every time and + // doesn't use MediaPipe's gpu buffer pool. + // TODO: ensure buffer pool is used when creating textures out of + // ImageFrame. + GlTexture CreateDestinationTexture(const ImageFrame& image_frame); + // The OpenGL name of the output framebuffer. GLuint framebuffer() const; @@ -196,7 +210,7 @@ class GlCalculatorHelper { // This class should be the main way to interface with GL memory within a single // calculator. This is the preferred way to utilize the memory pool inside of // the helper, because GlTexture manages efficiently releasing memory back into -// the pool. A GPU backed Image can be extracted from the unerlying +// the pool. A GPU backed Image can be extracted from the underlying // memory. class GlTexture { public: diff --git a/mediapipe/gpu/gl_context.cc b/mediapipe/gpu/gl_context.cc index 3244f22a..d7381bab 100644 --- a/mediapipe/gpu/gl_context.cc +++ b/mediapipe/gpu/gl_context.cc @@ -65,7 +65,7 @@ static void SetThreadName(const char* name) { #elif __APPLE__ pthread_setname_np(name); #endif - ANNOTATE_THREAD_NAME(name); + ABSL_ANNOTATE_THREAD_NAME(name); } GlContext::DedicatedThread::DedicatedThread() { diff --git a/mediapipe/gpu/gl_texture_buffer.h b/mediapipe/gpu/gl_texture_buffer.h index f785571a..7b914064 100644 --- a/mediapipe/gpu/gl_texture_buffer.h +++ b/mediapipe/gpu/gl_texture_buffer.h @@ -91,9 +91,9 @@ class GlTextureBuffer // TODO: turn into a single call? GLuint name() const { return name_; } GLenum target() const { return target_; } - int width() const { return width_; } - int height() const { return height_; } - GpuBufferFormat format() const { return format_; } + int width() const override { return width_; } + int height() const override { return height_; } + GpuBufferFormat format() const override { return format_; } GlTextureView GetReadView(internal::types, int plane) const override; diff --git a/mediapipe/gpu/image_frame_to_gpu_buffer_calculator.cc b/mediapipe/gpu/image_frame_to_gpu_buffer_calculator.cc index 2a8331db..a741e42a 100644 --- a/mediapipe/gpu/image_frame_to_gpu_buffer_calculator.cc +++ b/mediapipe/gpu/image_frame_to_gpu_buffer_calculator.cc @@ -71,11 +71,10 @@ absl::Status ImageFrameToGpuBufferCalculator::Process(CalculatorContext* cc) { #else const auto& input = cc->Inputs().Index(0).Get(); helper_.RunInGlContext([this, &input, &cc]() { - auto src = helper_.CreateSourceTexture(input); - auto output = src.GetFrame(); - glFlush(); + GlTexture dst = helper_.CreateDestinationTexture(input); + std::unique_ptr output = dst.GetFrame(); cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp()); - src.Release(); + dst.Release(); }); #endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER return absl::OkStatus(); diff --git a/mediapipe/model_maker/python/core/hyperparameters.py b/mediapipe/model_maker/python/core/hyperparameters.py index 3b3e3540..22471655 100644 --- a/mediapipe/model_maker/python/core/hyperparameters.py +++ b/mediapipe/model_maker/python/core/hyperparameters.py @@ -15,9 +15,12 @@ import dataclasses import tempfile - from typing import Optional +import tensorflow as tf + +from official.common import distribute_utils + @dataclasses.dataclass class BaseHParams: @@ -43,10 +46,10 @@ class BaseHParams: documentation for more details: https://www.tensorflow.org/api_docs/python/tf/distribute/Strategy. num_gpus: How many GPUs to use at each worker with the - DistributionStrategies API. The default is -1, which means utilize all - available GPUs. - tpu: The Cloud TPU to use for training. This should be either the name used - when creating the Cloud TPU, or a grpc://ip.address.of.tpu:8470 url. + DistributionStrategies API. The default is 0. + tpu: The TPU resource to be used for training. This should be either the + name used when creating the Cloud TPU, a grpc://ip.address.of.tpu:8470 + url, or an empty string if using a local TPU. """ # Parameters for train configuration @@ -63,5 +66,16 @@ class BaseHParams: # Parameters for hardware acceleration distribution_strategy: str = 'off' - num_gpus: int = -1 # default value of -1 means use all available GPUs + num_gpus: int = 0 tpu: str = '' + _strategy: tf.distribute.Strategy = dataclasses.field(init=False) + + def __post_init__(self): + self._strategy = distribute_utils.get_distribution_strategy( + distribution_strategy=self.distribution_strategy, + num_gpus=self.num_gpus, + tpu_address=self.tpu, + ) + + def get_strategy(self): + return self._strategy diff --git a/mediapipe/model_maker/python/text/text_classifier/model_spec_test.py b/mediapipe/model_maker/python/text/text_classifier/model_spec_test.py index d3daac54..a8d40558 100644 --- a/mediapipe/model_maker/python/text/text_classifier/model_spec_test.py +++ b/mediapipe/model_maker/python/text/text_classifier/model_spec_test.py @@ -85,8 +85,10 @@ class ModelSpecTest(tf.test.TestCase): steps_per_epoch=None, shuffle=False, distribution_strategy='off', - num_gpus=-1, - tpu='')) + num_gpus=0, + tpu='', + ), + ) def test_custom_bert_spec(self): custom_bert_classifier_options = ( diff --git a/mediapipe/model_maker/python/text/text_classifier/text_classifier.py b/mediapipe/model_maker/python/text/text_classifier/text_classifier.py index cd6ceb9b..c3dd48be 100644 --- a/mediapipe/model_maker/python/text/text_classifier/text_classifier.py +++ b/mediapipe/model_maker/python/text/text_classifier/text_classifier.py @@ -311,9 +311,11 @@ class _BertClassifier(TextClassifier): label_names: Sequence[str]): super().__init__(model_spec, hparams, label_names) self._model_options = model_options - self._loss_function = tf.keras.losses.SparseCategoricalCrossentropy() - self._metric_function = tf.keras.metrics.SparseCategoricalAccuracy( - "test_accuracy", dtype=tf.float32) + with self._hparams.get_strategy().scope(): + self._loss_function = tf.keras.losses.SparseCategoricalCrossentropy() + self._metric_function = tf.keras.metrics.SparseCategoricalAccuracy( + "test_accuracy", dtype=tf.float32 + ) self._text_preprocessor: preprocessor.BertClassifierPreprocessor = None @classmethod @@ -350,8 +352,9 @@ class _BertClassifier(TextClassifier): """ (processed_train_data, processed_validation_data) = ( self._load_and_run_preprocessor(train_data, validation_data)) - self._create_model() - self._create_optimizer(processed_train_data) + with self._hparams.get_strategy().scope(): + self._create_model() + self._create_optimizer(processed_train_data) self._train_model(processed_train_data, processed_validation_data) def _load_and_run_preprocessor( diff --git a/mediapipe/tasks/ios/BUILD b/mediapipe/tasks/ios/BUILD index fae60e24..29b0dd65 100644 --- a/mediapipe/tasks/ios/BUILD +++ b/mediapipe/tasks/ios/BUILD @@ -53,6 +53,7 @@ CALCULATORS_AND_GRAPHS = [ "//mediapipe/tasks/cc/text/text_classifier:text_classifier_graph", "//mediapipe/tasks/cc/text/text_embedder:text_embedder_graph", "//mediapipe/tasks/cc/vision/face_detector:face_detector_graph", + "//mediapipe/tasks/cc/vision/face_landmarker:face_landmarker_graph", "//mediapipe/tasks/cc/vision/image_classifier:image_classifier_graph", "//mediapipe/tasks/cc/vision/object_detector:object_detector_graph", ] @@ -80,6 +81,9 @@ strip_api_include_path_prefix( "//mediapipe/tasks/ios/vision/face_detector:sources/MPPFaceDetector.h", "//mediapipe/tasks/ios/vision/face_detector:sources/MPPFaceDetectorOptions.h", "//mediapipe/tasks/ios/vision/face_detector:sources/MPPFaceDetectorResult.h", + "//mediapipe/tasks/ios/vision/face_landmarker:sources/MPPFaceLandmarker.h", + "//mediapipe/tasks/ios/vision/face_landmarker:sources/MPPFaceLandmarkerOptions.h", + "//mediapipe/tasks/ios/vision/face_landmarker:sources/MPPFaceLandmarkerResult.h", "//mediapipe/tasks/ios/vision/image_classifier:sources/MPPImageClassifier.h", "//mediapipe/tasks/ios/vision/image_classifier:sources/MPPImageClassifierOptions.h", "//mediapipe/tasks/ios/vision/image_classifier:sources/MPPImageClassifierResult.h", @@ -164,6 +168,9 @@ apple_static_xcframework( ":MPPFaceDetector.h", ":MPPFaceDetectorOptions.h", ":MPPFaceDetectorResult.h", + ":MPPFaceLandmarker.h", + ":MPPFaceLandmarkerOptions.h", + ":MPPFaceLandmarkerResult.h", ":MPPImageClassifier.h", ":MPPImageClassifierOptions.h", ":MPPImageClassifierResult.h", @@ -173,6 +180,7 @@ apple_static_xcframework( ], deps = [ "//mediapipe/tasks/ios/vision/face_detector:MPPFaceDetector", + "//mediapipe/tasks/ios/vision/face_landmarker:MPPFaceLandmarker", "//mediapipe/tasks/ios/vision/image_classifier:MPPImageClassifier", "//mediapipe/tasks/ios/vision/object_detector:MPPObjectDetector", ], diff --git a/mediapipe/tasks/ios/MediaPipeTasksCommon.podspec.template b/mediapipe/tasks/ios/MediaPipeTasksCommon.podspec.template index de9a3fa2..1e622469 100644 --- a/mediapipe/tasks/ios/MediaPipeTasksCommon.podspec.template +++ b/mediapipe/tasks/ios/MediaPipeTasksCommon.podspec.template @@ -4,9 +4,9 @@ Pod::Spec.new do |s| s.authors = 'Google Inc.' s.license = { :type => 'Apache',:file => "LICENSE" } s.homepage = 'https://github.com/google/mediapipe' - s.source = { :http => '${MPP_DOWNLOAD_URL}' } + s.source = { :http => '${MPP_COMMON_DOWNLOAD_URL}' } s.summary = 'MediaPipe Task Library - Text' - s.description = 'The Natural Language APIs of the MediaPipe Task Library' + s.description = 'The common libraries of the MediaPipe Task Library' s.ios.deployment_target = '11.0' diff --git a/mediapipe/tasks/ios/MediaPipeTasksText.podspec.template b/mediapipe/tasks/ios/MediaPipeTasksText.podspec.template index 105f1c4f..f2f04bf7 100644 --- a/mediapipe/tasks/ios/MediaPipeTasksText.podspec.template +++ b/mediapipe/tasks/ios/MediaPipeTasksText.podspec.template @@ -4,7 +4,7 @@ Pod::Spec.new do |s| s.authors = 'Google Inc.' s.license = { :type => 'Apache',:file => "LICENSE" } s.homepage = 'https://github.com/google/mediapipe' - s.source = { :http => '${MPP_DOWNLOAD_URL}' } + s.source = { :http => '${MPP_TEXT_DOWNLOAD_URL}' } s.summary = 'MediaPipe Task Library - Text' s.description = 'The Natural Language APIs of the MediaPipe Task Library' diff --git a/mediapipe/tasks/ios/MediaPipeTasksVision.podspec.template b/mediapipe/tasks/ios/MediaPipeTasksVision.podspec.template index 4c0c0170..af63ba94 100644 --- a/mediapipe/tasks/ios/MediaPipeTasksVision.podspec.template +++ b/mediapipe/tasks/ios/MediaPipeTasksVision.podspec.template @@ -4,7 +4,7 @@ Pod::Spec.new do |s| s.authors = 'Google Inc.' s.license = { :type => 'Apache',:file => "LICENSE" } s.homepage = 'https://github.com/google/mediapipe' - s.source = { :http => '${MPP_DOWNLOAD_URL}' } + s.source = { :http => '${MPP_VISION_DOWNLOAD_URL}' } s.summary = 'MediaPipe Task Library - Vision' s.description = 'The Vision APIs of the MediaPipe Task Library' diff --git a/mediapipe/tasks/ios/test/vision/face_landmarker/BUILD b/mediapipe/tasks/ios/test/vision/face_landmarker/BUILD new file mode 100644 index 00000000..93ecc00f --- /dev/null +++ b/mediapipe/tasks/ios/test/vision/face_landmarker/BUILD @@ -0,0 +1,71 @@ +load("@build_bazel_rules_apple//apple:ios.bzl", "ios_unit_test") +load( + "//mediapipe/framework/tool:ios.bzl", + "MPP_TASK_MINIMUM_OS_VERSION", +) +load( + "@org_tensorflow//tensorflow/lite:special_rules.bzl", + "tflite_ios_lab_runner", +) + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +# Default tags for filtering iOS targets. Targets are restricted to Apple platforms. +TFL_DEFAULT_TAGS = [ + "apple", +] + +# Following sanitizer tests are not supported by iOS test targets. +TFL_DISABLED_SANITIZER_TAGS = [ + "noasan", + "nomsan", + "notsan", +] + +objc_library( + name = "MPPFaceLandmarkerObjcTestLibrary", + testonly = 1, + srcs = ["MPPFaceLandmarkerTests.mm"], + copts = [ + "-ObjC++", + "-std=c++17", + "-x objective-c++", + ], + data = [ + "//mediapipe/tasks/testdata/vision:test_images", + "//mediapipe/tasks/testdata/vision:test_models", + "//mediapipe/tasks/testdata/vision:test_protos", + ], + deps = [ + "//mediapipe/framework/formats:classification_cc_proto", + "//mediapipe/framework/formats:landmark_cc_proto", + "//mediapipe/framework/formats:matrix_data_cc_proto", + "//mediapipe/tasks/cc/vision/face_geometry/proto:face_geometry_cc_proto", + "//mediapipe/tasks/ios/common:MPPCommon", + "//mediapipe/tasks/ios/components/containers/utils:MPPClassificationResultHelpers", + "//mediapipe/tasks/ios/components/containers/utils:MPPDetectionHelpers", + "//mediapipe/tasks/ios/components/containers/utils:MPPLandmarkHelpers", + "//mediapipe/tasks/ios/test/vision/utils:MPPImageTestUtils", + "//mediapipe/tasks/ios/test/vision/utils:parse_proto_utils", + "//mediapipe/tasks/ios/vision/face_landmarker:MPPFaceLandmarker", + "//mediapipe/tasks/ios/vision/face_landmarker:MPPFaceLandmarkerResult", + "//third_party/apple_frameworks:UIKit", + ] + select({ + "//third_party:opencv_ios_sim_arm64_source_build": ["@ios_opencv_source//:opencv_xcframework"], + "//third_party:opencv_ios_arm64_source_build": ["@ios_opencv_source//:opencv_xcframework"], + "//third_party:opencv_ios_x86_64_source_build": ["@ios_opencv_source//:opencv_xcframework"], + "//conditions:default": ["@ios_opencv//:OpencvFramework"], + }), +) + +ios_unit_test( + name = "MPPFaceLandmarkerObjcTest", + minimum_os_version = MPP_TASK_MINIMUM_OS_VERSION, + runner = tflite_ios_lab_runner("IOS_LATEST"), + tags = TFL_DEFAULT_TAGS + TFL_DISABLED_SANITIZER_TAGS, + deps = [ + ":MPPFaceLandmarkerObjcTestLibrary", + ], +) diff --git a/mediapipe/tasks/ios/test/vision/face_landmarker/MPPFaceLandmarkerTests.mm b/mediapipe/tasks/ios/test/vision/face_landmarker/MPPFaceLandmarkerTests.mm new file mode 100644 index 00000000..f1d6033a --- /dev/null +++ b/mediapipe/tasks/ios/test/vision/face_landmarker/MPPFaceLandmarkerTests.mm @@ -0,0 +1,553 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import +#import +#import + +#include "mediapipe/framework/formats/classification.pb.h" +#include "mediapipe/framework/formats/landmark.pb.h" +#include "mediapipe/framework/formats/matrix_data.pb.h" +#include "mediapipe/tasks/cc/vision/face_geometry/proto/face_geometry.pb.h" +#import "mediapipe/tasks/ios/common/sources/MPPCommon.h" +#import "mediapipe/tasks/ios/components/containers/utils/sources/MPPClassificationResult+Helpers.h" +#import "mediapipe/tasks/ios/components/containers/utils/sources/MPPDetection+Helpers.h" +#import "mediapipe/tasks/ios/components/containers/utils/sources/MPPLandmark+Helpers.h" +#import "mediapipe/tasks/ios/test/vision/utils/sources/MPPImage+TestUtils.h" +#include "mediapipe/tasks/ios/test/vision/utils/sources/parse_proto_utils.h" +#import "mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarker.h" +#import "mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarkerResult.h" + +using NormalizedLandmarkListProto = ::mediapipe::NormalizedLandmarkList; +using ClassificationListProto = ::mediapipe::ClassificationList; +using FaceGeometryProto = ::mediapipe::tasks::vision::face_geometry::proto::FaceGeometry; +using ::mediapipe::tasks::ios::test::vision::utils::get_proto_from_pbtxt; + +static NSString *const kPbFileExtension = @"pbtxt"; + +typedef NSDictionary ResourceFileInfo; + +static ResourceFileInfo *const kPortraitImage = + @{@"name" : @"portrait", @"type" : @"jpg", @"orientation" : @(UIImageOrientationUp)}; +static ResourceFileInfo *const kPortraitRotatedImage = + @{@"name" : @"portrait_rotated", @"type" : @"jpg", @"orientation" : @(UIImageOrientationRight)}; +static ResourceFileInfo *const kCatImage = @{@"name" : @"cat", @"type" : @"jpg"}; +static ResourceFileInfo *const kPortraitExpectedLandmarksName = + @{@"name" : @"portrait_expected_face_landmarks", @"type" : kPbFileExtension}; +static ResourceFileInfo *const kPortraitExpectedBlendshapesName = + @{@"name" : @"portrait_expected_blendshapes", @"type" : kPbFileExtension}; +static ResourceFileInfo *const kPortraitExpectedGeometryName = + @{@"name" : @"portrait_expected_face_geometry", @"type" : kPbFileExtension}; +static NSString *const kFaceLandmarkerModelName = @"face_landmarker_v2"; +static NSString *const kFaceLandmarkerWithBlendshapesModelName = + @"face_landmarker_v2_with_blendshapes"; +static NSString *const kExpectedErrorDomain = @"com.google.mediapipe.tasks"; +static NSString *const kLiveStreamTestsDictFaceLandmarkerKey = @"face_landmarker"; +static NSString *const kLiveStreamTestsDictExpectationKey = @"expectation"; + +constexpr float kLandmarkErrorThreshold = 0.03f; +constexpr float kBlendshapesErrorThreshold = 0.1f; +constexpr float kFacialTransformationMatrixErrorThreshold = 0.2f; + +#define AssertEqualErrors(error, expectedError) \ + XCTAssertNotNil(error); \ + XCTAssertEqualObjects(error.domain, expectedError.domain); \ + XCTAssertEqual(error.code, expectedError.code); \ + XCTAssertEqualObjects(error.localizedDescription, expectedError.localizedDescription) + +@interface MPPFaceLandmarkerTests : XCTestCase { + NSDictionary *_liveStreamSucceedsTestDict; + NSDictionary *_outOfOrderTimestampTestDict; +} +@end + +@implementation MPPFaceLandmarkerTests + +#pragma mark General Tests + +- (void)testCreateFaceLandmarkerWithMissingModelPathFails { + NSString *modelPath = [MPPFaceLandmarkerTests filePathWithName:@"" extension:@""]; + + NSError *error = nil; + MPPFaceLandmarker *faceLandmarker = [[MPPFaceLandmarker alloc] initWithModelPath:modelPath + error:&error]; + XCTAssertNil(faceLandmarker); + + NSError *expectedError = [NSError + errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : + @"INVALID_ARGUMENT: ExternalFile must specify at least one of 'file_content', " + @"'file_name', 'file_pointer_meta' or 'file_descriptor_meta'." + }]; + AssertEqualErrors(error, expectedError); +} + +#pragma mark Image Mode Tests + +- (void)testDetectWithImageModeAndPotraitSucceeds { + NSString *modelPath = [MPPFaceLandmarkerTests filePathWithName:kFaceLandmarkerModelName + extension:@"task"]; + MPPFaceLandmarker *faceLandmarker = [[MPPFaceLandmarker alloc] initWithModelPath:modelPath + error:nil]; + NSArray *expectedLandmarks = + [MPPFaceLandmarkerTests expectedLandmarksFromFileInfo:kPortraitExpectedLandmarksName]; + [self assertResultsOfDetectInImageWithFileInfo:kPortraitImage + usingFaceLandmarker:faceLandmarker + containsExpectedLandmarks:expectedLandmarks + expectedBlendshapes:NULL + expectedTransformationMatrix:NULL]; +} + +- (void)testDetectWithImageModeAndPotraitAndFacialTransformationMatrixesSucceeds { + MPPFaceLandmarkerOptions *options = + [self faceLandmarkerOptionsWithModelName:kFaceLandmarkerModelName]; + options.outputFacialTransformationMatrixes = YES; + MPPFaceLandmarker *faceLandmarker = [[MPPFaceLandmarker alloc] initWithOptions:options error:nil]; + + NSArray *expectedLandmarks = + [MPPFaceLandmarkerTests expectedLandmarksFromFileInfo:kPortraitExpectedLandmarksName]; + MPPTransformMatrix *expectedTransformationMatrix = [MPPFaceLandmarkerTests + expectedTransformationMatrixFromFileInfo:kPortraitExpectedGeometryName]; + [self assertResultsOfDetectInImageWithFileInfo:kPortraitImage + usingFaceLandmarker:faceLandmarker + containsExpectedLandmarks:expectedLandmarks + expectedBlendshapes:NULL + expectedTransformationMatrix:expectedTransformationMatrix]; +} + +- (void)testDetectWithImageModeAndNoFaceSucceeds { + NSString *modelPath = [MPPFaceLandmarkerTests filePathWithName:kFaceLandmarkerModelName + extension:@"task"]; + MPPFaceLandmarker *faceLandmarker = [[MPPFaceLandmarker alloc] initWithModelPath:modelPath + error:nil]; + XCTAssertNotNil(faceLandmarker); + + NSError *error; + MPPImage *mppImage = [self imageWithFileInfo:kCatImage]; + MPPFaceLandmarkerResult *faceLandmarkerResult = [faceLandmarker detectInImage:mppImage + error:&error]; + XCTAssertNil(error); + XCTAssertNotNil(faceLandmarkerResult); + XCTAssertEqualObjects(faceLandmarkerResult.faceLandmarks, [NSArray array]); + XCTAssertEqualObjects(faceLandmarkerResult.faceBlendshapes, [NSArray array]); + XCTAssertEqualObjects(faceLandmarkerResult.facialTransformationMatrixes, [NSArray array]); +} + +#pragma mark Video Mode Tests + +- (void)testDetectWithVideoModeAndPotraitSucceeds { + MPPFaceLandmarkerOptions *options = + [self faceLandmarkerOptionsWithModelName:kFaceLandmarkerModelName]; + options.runningMode = MPPRunningModeVideo; + MPPFaceLandmarker *faceLandmarker = [[MPPFaceLandmarker alloc] initWithOptions:options error:nil]; + + MPPImage *image = [self imageWithFileInfo:kPortraitImage]; + NSArray *expectedLandmarks = + [MPPFaceLandmarkerTests expectedLandmarksFromFileInfo:kPortraitExpectedLandmarksName]; + for (int i = 0; i < 3; i++) { + MPPFaceLandmarkerResult *faceLandmarkerResult = [faceLandmarker detectInVideoFrame:image + timestampInMilliseconds:i + error:nil]; + [self assertFaceLandmarkerResult:faceLandmarkerResult + containsExpectedLandmarks:expectedLandmarks + expectedBlendshapes:NULL + expectedTransformationMatrix:NULL]; + } +} + +#pragma mark Live Stream Mode Tests + +- (void)testDetectWithLiveStreamModeAndPotraitSucceeds { + NSInteger iterationCount = 100; + + // Because of flow limiting, the callback might be invoked fewer than `iterationCount` times. An + // normal expectation will fail if expectation.fullfill() is not called + // `expectation.expectedFulfillmentCount` times. If `expectation.isInverted = true`, the test will + // only succeed if expectation is not fullfilled for the specified `expectedFulfillmentCount`. + // Since it is not possible to predict how many times the expectation is supposed to be + // fullfilled, `expectation.expectedFulfillmentCount` = `iterationCount` + 1 and + // `expectation.isInverted = true` ensures that test succeeds if expectation is fullfilled <= + // `iterationCount` times. + XCTestExpectation *expectation = [[XCTestExpectation alloc] + initWithDescription:@"detectWithOutOfOrderTimestampsAndLiveStream"]; + expectation.expectedFulfillmentCount = iterationCount + 1; + expectation.inverted = YES; + + MPPFaceLandmarkerOptions *options = + [self faceLandmarkerOptionsWithModelName:kFaceLandmarkerModelName]; + options.runningMode = MPPRunningModeLiveStream; + options.faceLandmarkerLiveStreamDelegate = self; + + MPPFaceLandmarker *faceLandmarker = [[MPPFaceLandmarker alloc] initWithOptions:options error:nil]; + MPPImage *image = [self imageWithFileInfo:kPortraitImage]; + + _liveStreamSucceedsTestDict = @{ + kLiveStreamTestsDictFaceLandmarkerKey : faceLandmarker, + kLiveStreamTestsDictExpectationKey : expectation + }; + + for (int i = 0; i < iterationCount; i++) { + XCTAssertTrue([faceLandmarker detectAsyncInImage:image timestampInMilliseconds:i error:nil]); + } + + NSTimeInterval timeout = 0.5f; + [self waitForExpectations:@[ expectation ] timeout:timeout]; +} + +- (void)testDetectWithOutOfOrderTimestampsAndLiveStreamModeFails { + MPPFaceLandmarkerOptions *options = + [self faceLandmarkerOptionsWithModelName:kFaceLandmarkerModelName]; + options.runningMode = MPPRunningModeLiveStream; + options.faceLandmarkerLiveStreamDelegate = self; + + XCTestExpectation *expectation = [[XCTestExpectation alloc] + initWithDescription:@"detectWithOutOfOrderTimestampsAndLiveStream"]; + expectation.expectedFulfillmentCount = 1; + + MPPFaceLandmarker *faceLandmarker = [[MPPFaceLandmarker alloc] initWithOptions:options error:nil]; + _liveStreamSucceedsTestDict = @{ + kLiveStreamTestsDictFaceLandmarkerKey : faceLandmarker, + kLiveStreamTestsDictExpectationKey : expectation + }; + + MPPImage *image = [self imageWithFileInfo:kPortraitImage]; + XCTAssertTrue([faceLandmarker detectAsyncInImage:image timestampInMilliseconds:1 error:nil]); + + NSError *error; + XCTAssertFalse([faceLandmarker detectAsyncInImage:image timestampInMilliseconds:0 error:&error]); + + NSError *expectedError = + [NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : + @"INVALID_ARGUMENT: Input timestamp must be monotonically increasing." + }]; + AssertEqualErrors(error, expectedError); + + NSTimeInterval timeout = 0.5f; + [self waitForExpectations:@[ expectation ] timeout:timeout]; +} + +#pragma mark Running Mode Tests + +- (void)testCreateFaceLandmarkerFailsWithDelegateInNonLiveStreamMode { + MPPRunningMode runningModesToTest[] = {MPPRunningModeImage, MPPRunningModeVideo}; + for (int i = 0; i < sizeof(runningModesToTest) / sizeof(runningModesToTest[0]); i++) { + MPPFaceLandmarkerOptions *options = + [self faceLandmarkerOptionsWithModelName:kFaceLandmarkerModelName]; + + options.runningMode = runningModesToTest[i]; + options.faceLandmarkerLiveStreamDelegate = self; + + [self + assertCreateFaceLandmarkerWithOptions:options + failsWithExpectedError: + [NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : + @"The vision task is in image or video mode. The " + @"delegate must not be set in the task's options." + }]]; + } +} + +- (void)testCreateFaceLandmarkerFailsWithMissingDelegateInLiveStreamMode { + MPPFaceLandmarkerOptions *options = + [self faceLandmarkerOptionsWithModelName:kFaceLandmarkerModelName]; + options.runningMode = MPPRunningModeLiveStream; + + [self assertCreateFaceLandmarkerWithOptions:options + failsWithExpectedError: + [NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : + @"The vision task is in live stream mode. An " + @"object must be set as the delegate of the task " + @"in its options to ensure asynchronous delivery " + @"of results." + }]]; +} + +- (void)testDetectFailsWithCallingWrongAPIInImageMode { + MPPFaceLandmarkerOptions *options = + [self faceLandmarkerOptionsWithModelName:kFaceLandmarkerModelName]; + MPPFaceLandmarker *faceLandmarker = [[MPPFaceLandmarker alloc] initWithOptions:options error:nil]; + + MPPImage *image = [self imageWithFileInfo:kPortraitImage]; + + NSError *liveStreamAPICallError; + XCTAssertFalse([faceLandmarker detectAsyncInImage:image + timestampInMilliseconds:0 + error:&liveStreamAPICallError]); + + NSError *expectedLiveStreamAPICallError = + [NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : @"The vision task is not initialized with live " + @"stream mode. Current Running Mode: Image" + }]; + AssertEqualErrors(liveStreamAPICallError, expectedLiveStreamAPICallError); + + NSError *videoAPICallError; + XCTAssertFalse([faceLandmarker detectInVideoFrame:image + timestampInMilliseconds:0 + error:&videoAPICallError]); + + NSError *expectedVideoAPICallError = + [NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : @"The vision task is not initialized with " + @"video mode. Current Running Mode: Image" + }]; + AssertEqualErrors(videoAPICallError, expectedVideoAPICallError); +} + +- (void)testDetectFailsWithCallingWrongAPIInVideoMode { + MPPFaceLandmarkerOptions *options = + [self faceLandmarkerOptionsWithModelName:kFaceLandmarkerModelName]; + options.runningMode = MPPRunningModeVideo; + + MPPFaceLandmarker *faceLandmarker = [[MPPFaceLandmarker alloc] initWithOptions:options error:nil]; + + MPPImage *image = [self imageWithFileInfo:kPortraitImage]; + NSError *liveStreamAPICallError; + XCTAssertFalse([faceLandmarker detectAsyncInImage:image + timestampInMilliseconds:0 + error:&liveStreamAPICallError]); + + NSError *expectedLiveStreamAPICallError = + [NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : @"The vision task is not initialized with live " + @"stream mode. Current Running Mode: Video" + }]; + AssertEqualErrors(liveStreamAPICallError, expectedLiveStreamAPICallError); + + NSError *imageAPICallError; + XCTAssertFalse([faceLandmarker detectInImage:image error:&imageAPICallError]); + + NSError *expectedImageAPICallError = + [NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : @"The vision task is not initialized with " + @"image mode. Current Running Mode: Video" + }]; + AssertEqualErrors(imageAPICallError, expectedImageAPICallError); +} + +- (void)testDetectFailsWithCallingWrongAPIInLiveStreamMode { + MPPFaceLandmarkerOptions *options = + [self faceLandmarkerOptionsWithModelName:kFaceLandmarkerModelName]; + options.runningMode = MPPRunningModeLiveStream; + options.faceLandmarkerLiveStreamDelegate = self; + MPPFaceLandmarker *faceLandmarker = [[MPPFaceLandmarker alloc] initWithOptions:options error:nil]; + + MPPImage *image = [self imageWithFileInfo:kPortraitImage]; + + NSError *imageAPICallError; + XCTAssertFalse([faceLandmarker detectInImage:image error:&imageAPICallError]); + + NSError *expectedImageAPICallError = + [NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : @"The vision task is not initialized with " + @"image mode. Current Running Mode: Live Stream" + }]; + AssertEqualErrors(imageAPICallError, expectedImageAPICallError); + + NSError *videoAPICallError; + XCTAssertFalse([faceLandmarker detectInVideoFrame:image + timestampInMilliseconds:0 + error:&videoAPICallError]); + + NSError *expectedVideoAPICallError = + [NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : @"The vision task is not initialized with " + @"video mode. Current Running Mode: Live Stream" + }]; + AssertEqualErrors(videoAPICallError, expectedVideoAPICallError); +} + +#pragma mark MPPFaceLandmarkerLiveStreamDelegate Methods +- (void)faceLandmarker:(MPPFaceLandmarker *)faceLandmarker + didFinishDetectionWithResult:(MPPFaceLandmarkerResult *)faceLandmarkerResult + timestampInMilliseconds:(NSInteger)timestampInMilliseconds + error:(NSError *)error { + NSArray *expectedLandmarks = + [MPPFaceLandmarkerTests expectedLandmarksFromFileInfo:kPortraitExpectedLandmarksName]; + [self assertFaceLandmarkerResult:faceLandmarkerResult + containsExpectedLandmarks:expectedLandmarks + expectedBlendshapes:NULL + expectedTransformationMatrix:NULL]; + + if (faceLandmarker == _outOfOrderTimestampTestDict[kLiveStreamTestsDictFaceLandmarkerKey]) { + [_outOfOrderTimestampTestDict[kLiveStreamTestsDictExpectationKey] fulfill]; + } else if (faceLandmarker == _liveStreamSucceedsTestDict[kLiveStreamTestsDictFaceLandmarkerKey]) { + [_liveStreamSucceedsTestDict[kLiveStreamTestsDictExpectationKey] fulfill]; + } +} + ++ (NSString *)filePathWithName:(NSString *)fileName extension:(NSString *)extension { + NSString *filePath = + [[NSBundle bundleForClass:[MPPFaceLandmarkerTests class]] pathForResource:fileName + ofType:extension]; + return filePath; +} + ++ (NSArray *)expectedLandmarksFromFileInfo:(NSDictionary *)fileInfo { + NSString *filePath = [self filePathWithName:fileInfo[@"name"] extension:fileInfo[@"type"]]; + NormalizedLandmarkListProto proto; + if (!get_proto_from_pbtxt([filePath UTF8String], proto).ok()) { + return nil; + } + NSMutableArray *landmarks = + [NSMutableArray arrayWithCapacity:(NSUInteger)proto.landmark_size()]; + for (const auto &landmarkProto : proto.landmark()) { + [landmarks addObject:[MPPNormalizedLandmark normalizedLandmarkWithProto:landmarkProto]]; + } + return landmarks; +} + ++ (MPPClassifications *)expectedBlendshapesFromFileInfo:(NSDictionary *)fileInfo { + NSString *filePath = [self filePathWithName:fileInfo[@"name"] extension:fileInfo[@"type"]]; + ClassificationListProto proto; + if (!get_proto_from_pbtxt([filePath UTF8String], proto).ok()) { + return nil; + } + return [MPPClassifications classificationsWithClassificationListProto:proto + headIndex:0 + headName:[NSString string]]; +} + ++ (MPPTransformMatrix *)expectedTransformationMatrixFromFileInfo:(NSDictionary *)fileInfo { + NSString *filePath = [self filePathWithName:fileInfo[@"name"] extension:fileInfo[@"type"]]; + FaceGeometryProto proto; + if (!get_proto_from_pbtxt([filePath UTF8String], proto).ok()) { + return nil; + } + return [[MPPTransformMatrix alloc] initWithData:proto.pose_transform_matrix().packed_data().data() + rows:proto.pose_transform_matrix().rows() + columns:proto.pose_transform_matrix().cols()]; +} + +- (void)assertFaceLandmarkerResult:(MPPFaceLandmarkerResult *)faceLandmarkerResult + containsExpectedLandmarks:(NSArray *)expectedLandmarks + expectedBlendshapes:(nullable MPPClassifications *)expectedBlendshapes + expectedTransformationMatrix:(nullable MPPTransformMatrix *)expectedTransformationMatrix { + NSArray *landmarks = faceLandmarkerResult.faceLandmarks[0]; + XCTAssertEqual(landmarks.count, expectedLandmarks.count); + for (int i = 0; i < landmarks.count; ++i) { + XCTAssertEqualWithAccuracy(landmarks[i].x, expectedLandmarks[i].x, kLandmarkErrorThreshold, + @"index i = %d", i); + XCTAssertEqualWithAccuracy(landmarks[i].y, expectedLandmarks[i].y, kLandmarkErrorThreshold, + @"index i = %d", i); + } + + if (expectedBlendshapes == NULL) { + XCTAssertEqualObjects(faceLandmarkerResult.faceBlendshapes, [NSArray array]); + } else { + MPPClassifications *blendshapes = faceLandmarkerResult.faceBlendshapes[0]; + NSArray *actualCategories = blendshapes.categories; + NSArray *expectedCategories = expectedBlendshapes.categories; + XCTAssertEqual(actualCategories.count, expectedCategories.count); + for (int i = 0; i < actualCategories.count; ++i) { + XCTAssertEqual(actualCategories[i].index, expectedCategories[i].index, @"index i = %d", i); + XCTAssertEqualWithAccuracy(actualCategories[i].score, expectedCategories[i].score, + kBlendshapesErrorThreshold, @"index i = %d", i); + XCTAssertEqualObjects(actualCategories[i].categoryName, expectedCategories[i].categoryName, + @"index i = %d", i); + XCTAssertEqualObjects(actualCategories[i].displayName, expectedCategories[i].displayName, + @"index i = %d", i); + } + } + + if (expectedTransformationMatrix == NULL) { + XCTAssertEqualObjects(faceLandmarkerResult.facialTransformationMatrixes, [NSArray array]); + } else { + MPPTransformMatrix *actualTransformationMatrix = + faceLandmarkerResult.facialTransformationMatrixes[0]; + XCTAssertEqual(actualTransformationMatrix.rows, expectedTransformationMatrix.rows); + XCTAssertEqual(actualTransformationMatrix.columns, expectedTransformationMatrix.columns); + for (int i = 0; i < actualTransformationMatrix.rows * actualTransformationMatrix.columns; ++i) { + XCTAssertEqualWithAccuracy(actualTransformationMatrix.data[i], + expectedTransformationMatrix.data[i], + kFacialTransformationMatrixErrorThreshold, @"index i = %d", i); + } + } +} + +#pragma mark Face Landmarker Initializers + +- (MPPFaceLandmarkerOptions *)faceLandmarkerOptionsWithModelName:(NSString *)modelName { + NSString *modelPath = [MPPFaceLandmarkerTests filePathWithName:modelName extension:@"task"]; + MPPFaceLandmarkerOptions *faceLandmarkerOptions = [[MPPFaceLandmarkerOptions alloc] init]; + faceLandmarkerOptions.baseOptions.modelAssetPath = modelPath; + return faceLandmarkerOptions; +} + +- (void)assertCreateFaceLandmarkerWithOptions:(MPPFaceLandmarkerOptions *)faceLandmarkerOptions + failsWithExpectedError:(NSError *)expectedError { + NSError *error = nil; + MPPFaceLandmarker *faceLandmarker = + [[MPPFaceLandmarker alloc] initWithOptions:faceLandmarkerOptions error:&error]; + XCTAssertNil(faceLandmarker); + AssertEqualErrors(error, expectedError); +} + +#pragma mark Assert Detection Results + +- (MPPImage *)imageWithFileInfo:(ResourceFileInfo *)fileInfo { + UIImageOrientation orientation = (UIImageOrientation)[fileInfo[@"orientation"] intValue]; + MPPImage *image = [MPPImage imageFromBundleWithClass:[MPPFaceLandmarkerTests class] + fileName:fileInfo[@"name"] + ofType:fileInfo[@"type"] + orientation:orientation]; + XCTAssertNotNil(image); + return image; +} + +- (void)assertResultsOfDetectInImageWithFileInfo:(ResourceFileInfo *)fileInfo + usingFaceLandmarker:(MPPFaceLandmarker *)faceLandmarker + containsExpectedLandmarks: + (NSArray *)expectedLandmarks + expectedBlendshapes:(nullable MPPClassifications *)expectedBlendshapes + expectedTransformationMatrix: + (nullable MPPTransformMatrix *)expectedTransformationMatrix { + MPPImage *mppImage = [self imageWithFileInfo:fileInfo]; + + NSError *error; + MPPFaceLandmarkerResult *faceLandmarkerResult = [faceLandmarker detectInImage:mppImage + error:&error]; + XCTAssertNil(error); + XCTAssertNotNil(faceLandmarkerResult); + + [self assertFaceLandmarkerResult:faceLandmarkerResult + containsExpectedLandmarks:expectedLandmarks + expectedBlendshapes:expectedBlendshapes + expectedTransformationMatrix:expectedTransformationMatrix]; +} + +@end diff --git a/mediapipe/tasks/ios/vision/face_landmarker/BUILD b/mediapipe/tasks/ios/vision/face_landmarker/BUILD index c4b17269..14f82237 100644 --- a/mediapipe/tasks/ios/vision/face_landmarker/BUILD +++ b/mediapipe/tasks/ios/vision/face_landmarker/BUILD @@ -37,7 +37,32 @@ objc_library( srcs = ["sources/MPPFaceLandmarkerOptions.m"], hdrs = ["sources/MPPFaceLandmarkerOptions.h"], deps = [ + ":MPPFaceLandmarkerResult", "//mediapipe/tasks/ios/core:MPPTaskOptions", "//mediapipe/tasks/ios/vision/core:MPPRunningMode", ], ) + +objc_library( + name = "MPPFaceLandmarker", + srcs = ["sources/MPPFaceLandmarker.m"], + hdrs = ["sources/MPPFaceLandmarker.h"], + copts = [ + "-ObjC++", + "-std=c++17", + "-x objective-c++", + ], + deps = [ + ":MPPFaceLandmarkerOptions", + ":MPPFaceLandmarkerResult", + "//mediapipe/tasks/cc/vision/face_landmarker:face_landmarker_graph", + "//mediapipe/tasks/ios/common/utils:MPPCommonUtils", + "//mediapipe/tasks/ios/common/utils:NSStringHelpers", + "//mediapipe/tasks/ios/core:MPPTaskInfo", + "//mediapipe/tasks/ios/vision/core:MPPImage", + "//mediapipe/tasks/ios/vision/core:MPPVisionPacketCreator", + "//mediapipe/tasks/ios/vision/core:MPPVisionTaskRunner", + "//mediapipe/tasks/ios/vision/face_landmarker/utils:MPPFaceLandmarkerOptionsHelpers", + "//mediapipe/tasks/ios/vision/face_landmarker/utils:MPPFaceLandmarkerResultHelpers", + ], +) diff --git a/mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarker.h b/mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarker.h new file mode 100644 index 00000000..fc8bed06 --- /dev/null +++ b/mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarker.h @@ -0,0 +1,156 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import + +#import "mediapipe/tasks/ios/vision/core/sources/MPPImage.h" +#import "mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarkerOptions.h" +#import "mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarkerResult.h" + +NS_ASSUME_NONNULL_BEGIN + +/** + * @brief Class that performs face landmark detection on images. + * + * The API expects a TFLite model with mandatory TFLite Model Metadata. + */ +NS_SWIFT_NAME(FaceLandmarker) +@interface MPPFaceLandmarker : NSObject + +/** + * Creates a new instance of `MPPFaceLandmarker` from an absolute path to a TensorFlow Lite model + * file stored locally on the device and the default `MPPFaceLandmarker`. + * + * @param modelPath An absolute path to a TensorFlow Lite model file stored locally on the device. + * @param error An optional error parameter populated when there is an error in initializing the + * face landmaker. + * + * @return A new instance of `MPPFaceLandmarker` with the given model path. `nil` if there is an + * error in initializing the face landmaker. + */ +- (nullable instancetype)initWithModelPath:(NSString *)modelPath error:(NSError **)error; + +/** + * Creates a new instance of `MPPFaceLandmarker` from the given `MPPFaceLandmarkerOptions`. + * + * @param options The options of type `MPPFaceLandmarkerOptions` to use for configuring the + * `MPPFaceLandmarker`. + * @param error An optional error parameter populated when there is an error in initializing the + * face landmaker. + * + * @return A new instance of `MPPFaceLandmarker` with the given options. `nil` if there is an error + * in initializing the face landmaker. + */ +- (nullable instancetype)initWithOptions:(MPPFaceLandmarkerOptions *)options + error:(NSError **)error NS_DESIGNATED_INITIALIZER; + +/** + * Performs face landmark detection on the provided MPPImage using the whole image as region of + * interest. Rotation will be applied according to the `orientation` property of the provided + * `MPPImage`. Only use this method when the `MPPFaceLandmarker` is created with + * `MPPRunningModeImage`. + * + * This method supports RGBA images. If your `MPPImage` has a source type of + * `MPPImageSourceTypePixelBuffer` or `MPPImageSourceTypeSampleBuffer`, the underlying pixel buffer + * must have one of the following pixel format types: + * 1. kCVPixelFormatType_32BGRA + * 2. kCVPixelFormatType_32RGBA + * + * If your `MPPImage` has a source type of `MPPImageSourceTypeImage` ensure that the color space is + * RGB with an Alpha channel. + * + * @param image The `MPPImage` on which face landmark detection is to be performed. + * @param error An optional error parameter populated when there is an error in performing face + * landmark detection on the input image. + * + * @return An `MPPFaceLandmarkerResult` that contains a list of landmarks. + */ +- (nullable MPPFaceLandmarkerResult *)detectInImage:(MPPImage *)image + error:(NSError **)error NS_SWIFT_NAME(detect(image:)); + +/** + * Performs face landmark detection on the provided video frame of type `MPPImage` using the whole + * image as region of interest. Rotation will be applied according to the `orientation` property of + * the provided `MPPImage`. Only use this method when the `MPPFaceLandmarker` is created with + * `MPPRunningModeVideo`. + * + * This method supports RGBA images. If your `MPPImage` has a source type of + * `MPPImageSourceTypePixelBuffer` or `MPPImageSourceTypeSampleBuffer`, the underlying pixel buffer + * must have one of the following pixel format types: + * 1. kCVPixelFormatType_32BGRA + * 2. kCVPixelFormatType_32RGBA + * + * If your `MPPImage` has a source type of `MPPImageSourceTypeImage` ensure that the color space is + * RGB with an Alpha channel. + * + * @param image The `MPPImage` on which face landmark detection is to be performed. + * @param timestampInMilliseconds The video frame's timestamp (in milliseconds). The input + * timestamps must be monotonically increasing. + * @param error An optional error parameter populated when there is an error in performing face + * landmark detection on the input image. + * + * @return An `MPPFaceLandmarkerResult` that contains a list of landmarks. + */ +- (nullable MPPFaceLandmarkerResult *)detectInVideoFrame:(MPPImage *)image + timestampInMilliseconds:(NSInteger)timestampInMilliseconds + error:(NSError **)error + NS_SWIFT_NAME(detect(videoFrame:timestampInMilliseconds:)); + +/** + * Sends live stream image data of type `MPPImage` to perform face landmark detection using the + * whole image as region of interest. Rotation will be applied according to the `orientation` + * property of the provided `MPPImage`. Only use this method when the `MPPFaceLandmarker` is created + * with `MPPRunningModeLiveStream`. + * + * The object which needs to be continuously notified of the available results of face + * detection must confirm to `MPPFaceLandmarkerLiveStreamDelegate` protocol and implement the + * `faceLandmarker:didFinishDetectionWithResult:timestampInMilliseconds:error:` delegate method. + * + * It's required to provide a timestamp (in milliseconds) to indicate when the input image is sent + * to the face detector. The input timestamps must be monotonically increasing. + * + * This method supports RGBA images. If your `MPPImage` has a source type of + * `MPPImageSourceTypePixelBuffer` or `MPPImageSourceTypeSampleBuffer`, the underlying pixel buffer + * must have one of the following pixel format types: + * 1. kCVPixelFormatType_32BGRA + * 2. kCVPixelFormatType_32RGBA + * + * If the input `MPPImage` has a source type of `MPPImageSourceTypeImage` ensure that the color + * space is RGB with an Alpha channel. + * + * If this method is used for classifying live camera frames using `AVFoundation`, ensure that you + * request `AVCaptureVideoDataOutput` to output frames in `kCMPixelFormat_32RGBA` using its + * `videoSettings` property. + * + * @param image A live stream image data of type `MPPImage` on which face landmark detection is to + * be performed. + * @param timestampInMilliseconds The timestamp (in milliseconds) which indicates when the input + * image is sent to the face detector. The input timestamps must be monotonically increasing. + * @param error An optional error parameter populated when there is an error when sending the input + * image to the graph. + * + * @return `YES` if the image was sent to the task successfully, otherwise `NO`. + */ +- (BOOL)detectAsyncInImage:(MPPImage *)image + timestampInMilliseconds:(NSInteger)timestampInMilliseconds + error:(NSError **)error + NS_SWIFT_NAME(detectAsync(image:timestampInMilliseconds:)); + +- (instancetype)init NS_UNAVAILABLE; + ++ (instancetype)new NS_UNAVAILABLE; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarker.m b/mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarker.m new file mode 100644 index 00000000..33dda7c0 --- /dev/null +++ b/mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarker.m @@ -0,0 +1,280 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarker.h" +#import + +#import "mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h" +#import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h" +#import "mediapipe/tasks/ios/core/sources/MPPTaskInfo.h" +#import "mediapipe/tasks/ios/vision/core/sources/MPPVisionPacketCreator.h" +#import "mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h" +#import "mediapipe/tasks/ios/vision/face_landmarker/utils/sources/MPPFaceLandmarkerOptions+Helpers.h" +#import "mediapipe/tasks/ios/vision/face_landmarker/utils/sources/MPPFaceLandmarkerResult+Helpers.h" + +using ::mediapipe::NormalizedRect; +using ::mediapipe::Packet; +using ::mediapipe::Timestamp; +using ::mediapipe::tasks::core::PacketMap; +using ::mediapipe::tasks::core::PacketsCallback; + +static constexpr int kMicrosecondsPerMillisecond = 1000; + +// Constants for the underlying MP Tasks Graph. See +// https://github.com/google/mediapipe/tree/master/mediapipe/tasks/cc/vision/face_landmarker/face_landmarker_graph.cc +static NSString *const kLandmarksOutStreamName = @"landmarks_out"; +static NSString *const kLandmarksOutTag = @"NORM_LANDMARKS"; +static NSString *const kBlendshapesOutStreamName = @"blendshapes_out"; +static NSString *const kBlendshapesOutTag = @"BLENDSHAPES"; +static NSString *const kFaceGeometryOutStreamName = @"face_geometry_out"; +static NSString *const kFaceGeometryOutTag = @"FACE_GEOMETRY"; +static NSString *const kNormRectStreamName = @"norm_rect_in"; +static NSString *const kNormRectTag = @"NORM_RECT"; +static NSString *const kImageInStreamName = @"image_in"; +static NSString *const kImageOutStreamName = @"image_out"; +static NSString *const kImageTag = @"IMAGE"; +static NSString *const kTaskGraphName = + @"mediapipe.tasks.vision.face_landmarker.FaceLandmarkerGraph"; +static NSString *const kTaskName = @"faceLandmarker"; + +#define InputPacketMap(imagePacket, normalizedRectPacket) \ + { \ + {kImageInStreamName.cppString, imagePacket}, { \ + kNormRectStreamName.cppString, normalizedRectPacket \ + } \ + } + +@interface MPPFaceLandmarker () { + /** iOS Vision Task Runner */ + MPPVisionTaskRunner *_visionTaskRunner; + /** + * The callback queue for the live stream delegate. This is only set if the user provides a live + * stream delegate. + */ + dispatch_queue_t _callbackQueue; + /** The user-provided live stream delegate if set. */ + __weak id _faceLandmarkerLiveStreamDelegate; +} +@end + +@implementation MPPFaceLandmarker + +- (instancetype)initWithOptions:(MPPFaceLandmarkerOptions *)options error:(NSError **)error { + self = [super init]; + if (self) { + NSArray *inputStreams = @[ + [NSString stringWithFormat:@"%@:%@", kImageTag, kImageInStreamName], + [NSString stringWithFormat:@"%@:%@", kNormRectTag, kNormRectStreamName] + ]; + + NSMutableArray *outputStreams = [NSMutableArray + arrayWithObjects:[NSString + stringWithFormat:@"%@:%@", kLandmarksOutTag, kLandmarksOutStreamName], + [NSString stringWithFormat:@"%@:%@", kImageTag, kImageOutStreamName], nil]; + if (options.outputFaceBlendshapes) { + [outputStreams addObject:[NSString stringWithFormat:@"%@:%@", kBlendshapesOutTag, + kBlendshapesOutStreamName]]; + } + if (options.outputFacialTransformationMatrixes) { + [outputStreams addObject:[NSString stringWithFormat:@"%@:%@", kFaceGeometryOutTag, + kFaceGeometryOutStreamName]]; + } + + MPPTaskInfo *taskInfo = + [[MPPTaskInfo alloc] initWithTaskGraphName:kTaskGraphName + inputStreams:inputStreams + outputStreams:outputStreams + taskOptions:options + enableFlowLimiting:options.runningMode == MPPRunningModeLiveStream + error:error]; + + if (!taskInfo) { + return nil; + } + + PacketsCallback packetsCallback = nullptr; + + if (options.faceLandmarkerLiveStreamDelegate) { + _faceLandmarkerLiveStreamDelegate = options.faceLandmarkerLiveStreamDelegate; + + // Create a private serial dispatch queue in which the delegate method will be called + // asynchronously. This is to ensure that if the client performs a long running operation in + // the delegate method, the queue on which the C++ callbacks is invoked is not blocked and is + // freed up to continue with its operations. + _callbackQueue = dispatch_queue_create( + [MPPVisionTaskRunner uniqueDispatchQueueNameWithSuffix:kTaskName], NULL); + + // Capturing `self` as weak in order to avoid `self` being kept in memory + // and cause a retain cycle, after self is set to `nil`. + MPPFaceLandmarker *__weak weakSelf = self; + packetsCallback = [weakSelf](absl::StatusOr liveStreamResult) { + [weakSelf processLiveStreamResult:liveStreamResult]; + }; + } + + _visionTaskRunner = + [[MPPVisionTaskRunner alloc] initWithCalculatorGraphConfig:[taskInfo generateGraphConfig] + runningMode:options.runningMode + packetsCallback:std::move(packetsCallback) + error:error]; + + if (!_visionTaskRunner) { + return nil; + } + } + + return self; +} + +- (instancetype)initWithModelPath:(NSString *)modelPath error:(NSError **)error { + MPPFaceLandmarkerOptions *options = [[MPPFaceLandmarkerOptions alloc] init]; + options.baseOptions.modelAssetPath = modelPath; + return [self initWithOptions:options error:error]; +} + +- (std::optional)inputPacketMapWithMPPImage:(MPPImage *)image + timestampInMilliseconds:(NSInteger)timestampInMilliseconds + error:(NSError **)error { + std::optional rect = + [_visionTaskRunner normalizedRectWithImageOrientation:image.orientation + imageSize:CGSizeMake(image.width, image.height) + error:error]; + if (!rect.has_value()) { + return std::nullopt; + } + + Packet imagePacket = [MPPVisionPacketCreator createPacketWithMPPImage:image + timestampInMilliseconds:timestampInMilliseconds + error:error]; + if (imagePacket.IsEmpty()) { + return std::nullopt; + } + + Packet normalizedRectPacket = + [MPPVisionPacketCreator createPacketWithNormalizedRect:*rect + timestampInMilliseconds:timestampInMilliseconds]; + + PacketMap inputPacketMap = InputPacketMap(imagePacket, normalizedRectPacket); + return inputPacketMap; +} + +- (nullable MPPFaceLandmarkerResult *)detectInImage:(MPPImage *)image error:(NSError **)error { + std::optional rect = + [_visionTaskRunner normalizedRectWithImageOrientation:image.orientation + imageSize:CGSizeMake(image.width, image.height) + error:error]; + if (!rect.has_value()) { + return nil; + } + + Packet imagePacket = [MPPVisionPacketCreator createPacketWithMPPImage:image error:error]; + if (imagePacket.IsEmpty()) { + return nil; + } + + Packet normalizedRectPacket = [MPPVisionPacketCreator createPacketWithNormalizedRect:*rect]; + + PacketMap inputPacketMap = InputPacketMap(imagePacket, normalizedRectPacket); + + std::optional outputPacketMap = [_visionTaskRunner processImagePacketMap:inputPacketMap + error:error]; + if (!outputPacketMap.has_value()) { + return nil; + } + + return [MPPFaceLandmarkerResult + faceLandmarkerResultWithLandmarksPacket:outputPacketMap + .value()[kLandmarksOutStreamName.cppString] + blendshapesPacket:outputPacketMap + .value()[kBlendshapesOutStreamName.cppString] + transformationMatrixesPacket:outputPacketMap + .value()[kFaceGeometryOutStreamName.cppString]]; +} + +- (nullable MPPFaceLandmarkerResult *)detectInVideoFrame:(MPPImage *)image + timestampInMilliseconds:(NSInteger)timestampInMilliseconds + error:(NSError **)error { + std::optional inputPacketMap = [self inputPacketMapWithMPPImage:image + timestampInMilliseconds:timestampInMilliseconds + error:error]; + if (!inputPacketMap.has_value()) { + return nil; + } + + std::optional outputPacketMap = + [_visionTaskRunner processVideoFramePacketMap:*inputPacketMap error:error]; + if (!outputPacketMap.has_value()) { + return nil; + } + + return [MPPFaceLandmarkerResult + faceLandmarkerResultWithLandmarksPacket:outputPacketMap + .value()[kLandmarksOutStreamName.cppString] + blendshapesPacket:outputPacketMap + .value()[kBlendshapesOutStreamName.cppString] + transformationMatrixesPacket:outputPacketMap + .value()[kFaceGeometryOutStreamName.cppString]]; +} + +- (BOOL)detectAsyncInImage:(MPPImage *)image + timestampInMilliseconds:(NSInteger)timestampInMilliseconds + error:(NSError **)error { + std::optional inputPacketMap = [self inputPacketMapWithMPPImage:image + timestampInMilliseconds:timestampInMilliseconds + error:error]; + if (!inputPacketMap.has_value()) { + return NO; + } + + return [_visionTaskRunner processLiveStreamPacketMap:*inputPacketMap error:error]; +} + +- (void)processLiveStreamResult:(absl::StatusOr)liveStreamResult { + NSError *callbackError; + if (![MPPCommonUtils checkCppError:liveStreamResult.status() toError:&callbackError]) { + dispatch_async(_callbackQueue, ^{ + [_faceLandmarkerLiveStreamDelegate faceLandmarker:self + didFinishDetectionWithResult:nil + timestampInMilliseconds:Timestamp::Unset().Value() + error:callbackError]; + }); + return; + } + + PacketMap &outputPacketMap = *liveStreamResult; + if (outputPacketMap[kImageOutStreamName.cppString].IsEmpty()) { + // The graph did not return a result. We therefore do not raise the user callback. This mirrors + // returning `nil` in the other methods and is acceptable for the live stream delegate since + // it is expected that we drop frames and don't return results for every input. + return; + } + + MPPFaceLandmarkerResult *result = [MPPFaceLandmarkerResult + faceLandmarkerResultWithLandmarksPacket:outputPacketMap[kLandmarksOutStreamName.cppString] + blendshapesPacket:outputPacketMap[kBlendshapesOutStreamName.cppString] + transformationMatrixesPacket:outputPacketMap[kFaceGeometryOutStreamName + .cppString]]; + + NSInteger timeStampInMilliseconds = + outputPacketMap[kImageOutStreamName.cppString].Timestamp().Value() / + kMicrosecondsPerMillisecond; + dispatch_async(_callbackQueue, ^{ + [_faceLandmarkerLiveStreamDelegate faceLandmarker:self + didFinishDetectionWithResult:result + timestampInMilliseconds:timeStampInMilliseconds + error:callbackError]; + }); +} + +@end diff --git a/mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarkerOptions.h b/mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarkerOptions.h index 873858a5..23b423ad 100644 --- a/mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarkerOptions.h +++ b/mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarkerOptions.h @@ -16,9 +16,45 @@ #import "mediapipe/tasks/ios/core/sources/MPPTaskOptions.h" #import "mediapipe/tasks/ios/vision/core/sources/MPPRunningMode.h" +#import "mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarkerResult.h" NS_ASSUME_NONNULL_BEGIN +@class MPPFaceLandmarker; + +/** + * This protocol defines an interface for the delegates of `MPPFaceLandmarker` face to receive + * results of performing asynchronous face detection on images (i.e, when `runningMode` = + * `MPPRunningModeLiveStream`). + * + * The delegate of `MPPFaceLandmarker` must adopt `MPPFaceLandmarkerLiveStreamDelegate` protocol. + * The methods in this protocol are optional. + */ +NS_SWIFT_NAME(FaceDetectorLiveStreamDelegate) +@protocol MPPFaceLandmarkerLiveStreamDelegate + +/** + * This method notifies a delegate that the results of asynchronous face detection of + * an image submitted to the `MPPFaceLandmarker` is available. + * + * This method is called on a private serial dispatch queue created by the `MPPFaceLandmarker` + * for performing the asynchronous delegates calls. + * + * @param faceLandmarker The face landmarker which performed the face landmark detctions. + * This is useful to test equality when there are multiple instances of `MPPFaceLandmarker`. + * @param result The `MPPFaceLandmarkerResult` object that contains a list of landmarks. + * @param timestampInMilliseconds The timestamp (in milliseconds) which indicates when the input + * image was sent to the face detector. + * @param error An optional error parameter populated when there is an error in performing face + * detection on the input live stream image data. + */ +- (void)faceLandmarker:(MPPFaceLandmarker *)faceLandmarker + didFinishDetectionWithResult:(nullable MPPFaceLandmarkerResult *)result + timestampInMilliseconds:(NSInteger)timestampInMilliseconds + error:(nullable NSError *)error + NS_SWIFT_NAME(faceLandmarker(_:didFinishDetection:timestampInMilliseconds:error:)); +@end + /** Options for setting up a `MPPFaceLandmarker`. */ NS_SWIFT_NAME(FaceLandmarkerOptions) @interface MPPFaceLandmarkerOptions : MPPTaskOptions @@ -34,6 +70,15 @@ NS_SWIFT_NAME(FaceLandmarkerOptions) */ @property(nonatomic) MPPRunningMode runningMode; +/** + * An object that confirms to `MPPFaceLandmarkerLiveStreamDelegate` protocol. This object must + * implement `faceLandmarker:didFinishDetectionWithResult:timestampInMilliseconds:error:` to receive + * the results of performing asynchronous face landmark detection on images (i.e, when `runningMode` + * = `MPPRunningModeLiveStream`). + */ +@property(nonatomic, weak, nullable) id + faceLandmarkerLiveStreamDelegate; + /** The maximum number of faces can be detected by the FaceLandmarker. Defaults to 1. */ @property(nonatomic) NSInteger numFaces; @@ -59,6 +104,13 @@ NS_SWIFT_NAME(FaceLandmarkerOptions) */ @property(nonatomic) BOOL outputFaceBlendshapes; +/** + * Whether FaceLandmarker outputs facial transformation_matrix. Facial transformation matrix is used + * to transform the face landmarks in canonical face to the detected face, so that users can apply + * face effects on the detected landmarks. + */ +@property(nonatomic) BOOL outputFacialTransformationMatrixes; + @end NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarkerOptions.m b/mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarkerOptions.m index 80350d12..ebef092f 100644 --- a/mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarkerOptions.m +++ b/mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarkerOptions.m @@ -24,6 +24,8 @@ _minFacePresenceConfidence = 0.5f; _minTrackingConfidence = 0.5f; _outputFaceBlendshapes = NO; + _outputFacialTransformationMatrixes = NO; + _outputFacialTransformationMatrixes = NO; } return self; } @@ -36,6 +38,9 @@ faceLandmarkerOptions.minFacePresenceConfidence = self.minFacePresenceConfidence; faceLandmarkerOptions.minTrackingConfidence = self.minTrackingConfidence; faceLandmarkerOptions.outputFaceBlendshapes = self.outputFaceBlendshapes; + faceLandmarkerOptions.outputFacialTransformationMatrixes = + self.outputFacialTransformationMatrixes; + faceLandmarkerOptions.faceLandmarkerLiveStreamDelegate = self.faceLandmarkerLiveStreamDelegate; return faceLandmarkerOptions; } diff --git a/mediapipe/tasks/ios/vision/image_classifier/BUILD b/mediapipe/tasks/ios/vision/image_classifier/BUILD index 69490f71..cf89249c 100644 --- a/mediapipe/tasks/ios/vision/image_classifier/BUILD +++ b/mediapipe/tasks/ios/vision/image_classifier/BUILD @@ -55,7 +55,6 @@ objc_library( "//mediapipe/tasks/ios/common/utils:MPPCommonUtils", "//mediapipe/tasks/ios/common/utils:NSStringHelpers", "//mediapipe/tasks/ios/core:MPPTaskInfo", - "//mediapipe/tasks/ios/core:MPPTaskOptions", "//mediapipe/tasks/ios/vision/core:MPPImage", "//mediapipe/tasks/ios/vision/core:MPPVisionPacketCreator", "//mediapipe/tasks/ios/vision/core:MPPVisionTaskRunner", diff --git a/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifier.h b/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifier.h index 398236bb..6b81a240 100644 --- a/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifier.h +++ b/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifier.h @@ -14,7 +14,6 @@ #import -#import "mediapipe/tasks/ios/core/sources/MPPTaskOptions.h" #import "mediapipe/tasks/ios/vision/core/sources/MPPImage.h" #import "mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifierOptions.h" #import "mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifierResult.h" diff --git a/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifier.mm b/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifier.mm index 10f6fc26..5d2595cd 100644 --- a/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifier.mm +++ b/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifier.mm @@ -55,6 +55,7 @@ static const int kMicroSecondsPerMilliSecond = 1000; @interface MPPImageClassifier () { /** iOS Vision Task Runner */ MPPVisionTaskRunner *_visionTaskRunner; + dispatch_queue_t _callbackQueue; } @property(nonatomic, weak) id imageClassifierLiveStreamDelegate; @@ -62,6 +63,44 @@ static const int kMicroSecondsPerMilliSecond = 1000; @implementation MPPImageClassifier +- (void)processLiveStreamResult:(absl::StatusOr)liveStreamResult { + if (![self.imageClassifierLiveStreamDelegate + respondsToSelector:@selector + (imageClassifier:didFinishClassificationWithResult:timestampInMilliseconds:error:)]) { + return; + } + + NSError *callbackError = nil; + if (![MPPCommonUtils checkCppError:liveStreamResult.status() toError:&callbackError]) { + dispatch_async(_callbackQueue, ^{ + [self.imageClassifierLiveStreamDelegate imageClassifier:self + didFinishClassificationWithResult:nil + timestampInMilliseconds:Timestamp::Unset().Value() + error:callbackError]; + }); + return; + } + + PacketMap &outputPacketMap = liveStreamResult.value(); + if (outputPacketMap[kImageOutStreamName.cppString].IsEmpty()) { + return; + } + + MPPImageClassifierResult *result = [MPPImageClassifierResult + imageClassifierResultWithClassificationsPacket:outputPacketMap[kClassificationsStreamName + .cppString]]; + + NSInteger timeStampInMilliseconds = + outputPacketMap[kImageOutStreamName.cppString].Timestamp().Value() / + kMicroSecondsPerMilliSecond; + dispatch_async(_callbackQueue, ^{ + [self.imageClassifierLiveStreamDelegate imageClassifier:self + didFinishClassificationWithResult:result + timestampInMilliseconds:timeStampInMilliseconds + error:callbackError]; + }); +} + - (instancetype)initWithOptions:(MPPImageClassifierOptions *)options error:(NSError **)error { self = [super init]; if (self) { @@ -88,56 +127,19 @@ static const int kMicroSecondsPerMilliSecond = 1000; if (options.imageClassifierLiveStreamDelegate) { _imageClassifierLiveStreamDelegate = options.imageClassifierLiveStreamDelegate; - // Capturing `self` as weak in order to avoid `self` being kept in memory - // and cause a retain cycle, after self is set to `nil`. - MPPImageClassifier *__weak weakSelf = self; // Create a private serial dispatch queue in which the deleagte method will be called // asynchronously. This is to ensure that if the client performs a long running operation in // the delegate method, the queue on which the C++ callbacks is invoked is not blocked and is // freed up to continue with its operations. - const char *queueName = [MPPVisionTaskRunner uniqueDispatchQueueNameWithSuffix:kTaskName]; - dispatch_queue_t callbackQueue = dispatch_queue_create(queueName, NULL); - packetsCallback = [=](absl::StatusOr status_or_packets) { - if (!weakSelf) { - return; - } - if (![weakSelf.imageClassifierLiveStreamDelegate - respondsToSelector:@selector - (imageClassifier: - didFinishClassificationWithResult:timestampInMilliseconds:error:)]) { - return; - } + _callbackQueue = dispatch_queue_create( + [MPPVisionTaskRunner uniqueDispatchQueueNameWithSuffix:kTaskName], NULL); - NSError *callbackError = nil; - if (![MPPCommonUtils checkCppError:status_or_packets.status() toError:&callbackError]) { - dispatch_async(callbackQueue, ^{ - [weakSelf.imageClassifierLiveStreamDelegate imageClassifier:weakSelf - didFinishClassificationWithResult:nil - timestampInMilliseconds:Timestamp::Unset().Value() - error:callbackError]; - }); - return; - } - - PacketMap &outputPacketMap = status_or_packets.value(); - if (outputPacketMap[kImageOutStreamName.cppString].IsEmpty()) { - return; - } - - MPPImageClassifierResult *result = - [MPPImageClassifierResult imageClassifierResultWithClassificationsPacket: - outputPacketMap[kClassificationsStreamName.cppString]]; - - NSInteger timeStampInMilliseconds = - outputPacketMap[kImageOutStreamName.cppString].Timestamp().Value() / - kMicroSecondsPerMilliSecond; - dispatch_async(callbackQueue, ^{ - [weakSelf.imageClassifierLiveStreamDelegate imageClassifier:weakSelf - didFinishClassificationWithResult:result - timestampInMilliseconds:timeStampInMilliseconds - error:callbackError]; - }); + // Capturing `self` as weak in order to avoid `self` being kept in memory + // and cause a retain cycle, after self is set to `nil`. + MPPImageClassifier *__weak weakSelf = self; + packetsCallback = [=](absl::StatusOr liveStreamResult) { + [weakSelf processLiveStreamResult:liveStreamResult]; }; } diff --git a/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.mm b/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.mm index b7924996..52648af5 100644 --- a/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.mm +++ b/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.mm @@ -50,12 +50,51 @@ static NSString *const kTaskName = @"objectDetector"; @interface MPPObjectDetector () { /** iOS Vision Task Runner */ MPPVisionTaskRunner *_visionTaskRunner; + dispatch_queue_t _callbackQueue; } @property(nonatomic, weak) id objectDetectorLiveStreamDelegate; @end @implementation MPPObjectDetector +- (void)processLiveStreamResult:(absl::StatusOr)liveStreamResult { + if (![self.objectDetectorLiveStreamDelegate + respondsToSelector:@selector(objectDetector: + didFinishDetectionWithResult:timestampInMilliseconds:error:)]) { + return; + } + + NSError *callbackError = nil; + if (![MPPCommonUtils checkCppError:liveStreamResult.status() toError:&callbackError]) { + dispatch_async(_callbackQueue, ^{ + [self.objectDetectorLiveStreamDelegate objectDetector:self + didFinishDetectionWithResult:nil + timestampInMilliseconds:Timestamp::Unset().Value() + error:callbackError]; + }); + return; + } + + PacketMap &outputPacketMap = liveStreamResult.value(); + if (outputPacketMap[kImageOutStreamName.cppString].IsEmpty()) { + return; + } + + MPPObjectDetectorResult *result = [MPPObjectDetectorResult + objectDetectorResultWithDetectionsPacket: + outputPacketMap[kDetectionsStreamName.cppString]]; + + NSInteger timeStampInMilliseconds = + outputPacketMap[kImageOutStreamName.cppString].Timestamp().Value() / + kMicroSecondsPerMilliSecond; + dispatch_async(_callbackQueue, ^{ + [self.objectDetectorLiveStreamDelegate objectDetector:self + didFinishDetectionWithResult:result + timestampInMilliseconds:timeStampInMilliseconds + error:callbackError]; + }); +} + - (instancetype)initWithOptions:(MPPObjectDetectorOptions *)options error:(NSError **)error { self = [super init]; if (self) { @@ -82,55 +121,18 @@ static NSString *const kTaskName = @"objectDetector"; if (options.objectDetectorLiveStreamDelegate) { _objectDetectorLiveStreamDelegate = options.objectDetectorLiveStreamDelegate; - // Capturing `self` as weak in order to avoid `self` being kept in memory - // and cause a retain cycle, after self is set to `nil`. - MPPObjectDetector *__weak weakSelf = self; - // Create a private serial dispatch queue in which the delegate method will be called // asynchronously. This is to ensure that if the client performs a long running operation in // the delegate method, the queue on which the C++ callbacks is invoked is not blocked and is // freed up to continue with its operations. - dispatch_queue_t callbackQueue = dispatch_queue_create( + _callbackQueue = dispatch_queue_create( [MPPVisionTaskRunner uniqueDispatchQueueNameWithSuffix:kTaskName], NULL); - packetsCallback = [=](absl::StatusOr statusOrPackets) { - if (!weakSelf) { - return; - } - if (![weakSelf.objectDetectorLiveStreamDelegate - respondsToSelector:@selector - (objectDetector:didFinishDetectionWithResult:timestampInMilliseconds:error:)]) { - return; - } - NSError *callbackError = nil; - if (![MPPCommonUtils checkCppError:statusOrPackets.status() toError:&callbackError]) { - dispatch_async(callbackQueue, ^{ - [weakSelf.objectDetectorLiveStreamDelegate objectDetector:weakSelf - didFinishDetectionWithResult:nil - timestampInMilliseconds:Timestamp::Unset().Value() - error:callbackError]; - }); - return; - } - - PacketMap &outputPacketMap = statusOrPackets.value(); - if (outputPacketMap[kImageOutStreamName.cppString].IsEmpty()) { - return; - } - - MPPObjectDetectorResult *result = [MPPObjectDetectorResult - objectDetectorResultWithDetectionsPacket:statusOrPackets - .value()[kDetectionsStreamName.cppString]]; - - NSInteger timeStampInMilliseconds = - outputPacketMap[kImageOutStreamName.cppString].Timestamp().Value() / - kMicroSecondsPerMilliSecond; - dispatch_async(callbackQueue, ^{ - [weakSelf.objectDetectorLiveStreamDelegate objectDetector:weakSelf - didFinishDetectionWithResult:result - timestampInMilliseconds:timeStampInMilliseconds - error:callbackError]; - }); + // Capturing `self` as weak in order to avoid `self` being kept in memory + // and cause a retain cycle, after self is set to `nil`. + MPPObjectDetector *__weak weakSelf = self; + packetsCallback = [=](absl::StatusOr liveStreamResult) { + [weakSelf processLiveStreamResult:liveStreamResult]; }; } diff --git a/mediapipe/util/tracking/region_flow_computation.cc b/mediapipe/util/tracking/region_flow_computation.cc index 7f67c393..b6704cc6 100644 --- a/mediapipe/util/tracking/region_flow_computation.cc +++ b/mediapipe/util/tracking/region_flow_computation.cc @@ -2096,6 +2096,12 @@ bool RegionFlowComputation::GainCorrectFrame(const cv::Mat& reference_frame, void RegionFlowComputation::WideBaselineMatchFeatures( FrameTrackingData* from_data_ptr, FrameTrackingData* to_data_ptr, TrackedFeatureList* results) { +#if (defined(__ANDROID__) || defined(__APPLE__) || defined(__EMSCRIPTEN__)) && \ + !defined(CV_WRAPPER_3X) + LOG(FATAL) << "Supported on only with OpenCV 3.0. " + << "Use bazel build flag : --define CV_WRAPPER=3X"; +#else // (defined(__ANDROID__) || defined(__APPLE__) || + // defined(__EMSCRIPTEN__)) && !defined(CV_WRAPPER_3X) results->clear(); const auto& frame1 = from_data_ptr->frame; @@ -2168,6 +2174,8 @@ void RegionFlowComputation::WideBaselineMatchFeatures( results->push_back(tracked_feature); } } +#endif // (defined(__ANDROID__) || defined(__APPLE__) || + // defined(__EMSCRIPTEN__)) && !defined(CV_WRAPPER_3X) } void RegionFlowComputation::RemoveAbsentFeatures( diff --git a/third_party/wasm_files.bzl b/third_party/wasm_files.bzl index 4c76e7bd..8ef0a71a 100644 --- a/third_party/wasm_files.bzl +++ b/third_party/wasm_files.bzl @@ -12,72 +12,72 @@ def wasm_files(): http_file( name = "com_google_mediapipe_wasm_audio_wasm_internal_js", - sha256 = "b07bf0eda990b19c48f2b51c358bb281f40a7c3002f50f16986fe96f68103ac1", - urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/audio_wasm_internal.js?generation=1683564589395847"], + sha256 = "0d66a26fa5ca638c54ec3e5bffb50aec74ee0880b108d4b5f7d316e9ae36cc9a", + urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/audio_wasm_internal.js?generation=1685638894464709"], ) http_file( name = "com_google_mediapipe_wasm_audio_wasm_internal_wasm", - sha256 = "725fa5d13fdce79beaccb287b24d79d32e7bfb40f2cc51ef7f26a8dd8dec993c", - urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/audio_wasm_internal.wasm?generation=1683564592604733"], + sha256 = "014963d19ef6b1f25720379c3df07a6e08b24894ada4938d45b1256e97739318", + urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/audio_wasm_internal.wasm?generation=1685638897160853"], ) http_file( name = "com_google_mediapipe_wasm_audio_wasm_nosimd_internal_js", - sha256 = "f2917690317ae381782c8f4fb17d40f7d0b8e340fb490a604a37959ecee637c8", - urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/audio_wasm_nosimd_internal.js?generation=1683564595399509"], + sha256 = "f03d4826c251783bfc1fb8b82b2d08c00b2e3cb2efcc606305eb210f09fc686b", + urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/audio_wasm_nosimd_internal.js?generation=1685638899477366"], ) http_file( name = "com_google_mediapipe_wasm_audio_wasm_nosimd_internal_wasm", - sha256 = "908f6ce2420b5e88770b61b20b200cb3cd62a1c727cf0a134aa645351eaa1350", - urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/audio_wasm_nosimd_internal.wasm?generation=1683564598602444"], + sha256 = "36972cf62138bcb5fde37a1fecce334a86b0261eefc1f1daa17b4b8acdc784b4", + urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/audio_wasm_nosimd_internal.wasm?generation=1685638901926088"], ) http_file( name = "com_google_mediapipe_wasm_text_wasm_internal_js", - sha256 = "64c4a3927e732b99473b072228130b922427b2aba16d64863579928df16a7946", - urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/text_wasm_internal.js?generation=1683564601486513"], + sha256 = "5745360da942f3bcb585547e8720cb11f19793e68851b119b8f9ea22b120fd06", + urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/text_wasm_internal.js?generation=1685638904214551"], ) http_file( name = "com_google_mediapipe_wasm_text_wasm_internal_wasm", - sha256 = "c6e3027f4a7b1fd11d5ebbd8254f168636658e7a5d19a293e900f19497e48d5e", - urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/text_wasm_internal.wasm?generation=1683564604687320"], + sha256 = "b6d8b03fa7fc3e969febfcb63e3db2de900f1f54b82bf2205f02d865fc4790b2", + urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/text_wasm_internal.wasm?generation=1685638906864568"], ) http_file( name = "com_google_mediapipe_wasm_text_wasm_nosimd_internal_js", - sha256 = "af59aacaddc076ca9e4ea139d4f440b5b114576e72ab69e50f0f501e0e0c07e5", - urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/text_wasm_nosimd_internal.js?generation=1683564606939116"], + sha256 = "837ca361044441e6202858b4a9d94b3296c8440099b40e6dafb1efcce76a8f63", + urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/text_wasm_nosimd_internal.js?generation=1685638909139832"], ) http_file( name = "com_google_mediapipe_wasm_text_wasm_nosimd_internal_wasm", - sha256 = "d313497c003b6e00670664463fbbd9f5a2388946fe3d132c2794dd87cb91beb0", - urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/text_wasm_nosimd_internal.wasm?generation=1683564609634068"], + sha256 = "507f4089f4a2cf8fe7fb61f48e180f3f86d5e8057fc60ef24c77aae724eb66ba", + urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/text_wasm_nosimd_internal.wasm?generation=1685638911843312"], ) http_file( name = "com_google_mediapipe_wasm_vision_wasm_internal_js", - sha256 = "efebf9d676d8828c31e7c0d9718c5c80de8a3de084e97aa3ea5472a5346c518e", - urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/vision_wasm_internal.js?generation=1683564611613681"], + sha256 = "82de7a40fdb14833b5ceaeb1ebf219421dbb06ba5e525204737dec196161420d", + urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/vision_wasm_internal.js?generation=1685638914190745"], ) http_file( name = "com_google_mediapipe_wasm_vision_wasm_internal_wasm", - sha256 = "0431b3bacfcb26d91d800450216b305b9378f4e063d78c2e85a944aba432e0dd", - urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/vision_wasm_internal.wasm?generation=1683564614567083"], + sha256 = "d06ac49f4c156cf0c24ef62387b13e48b67476e7f04a423889c59ee835c460f2", + urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/vision_wasm_internal.wasm?generation=1685638917012370"], ) http_file( name = "com_google_mediapipe_wasm_vision_wasm_nosimd_internal_js", - sha256 = "e51fa49f60493d7122d26e6fcb45d4031a3247a05d83b3f62e5155653a89d8f8", - urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/vision_wasm_nosimd_internal.js?generation=1683564616879456"], + sha256 = "fff428ef91d8cc936f9c3ec81750f5e7ee3c20bc0c76677eb5d8d4d010d2fac0", + urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/vision_wasm_nosimd_internal.js?generation=1685638919406810"], ) http_file( name = "com_google_mediapipe_wasm_vision_wasm_nosimd_internal_wasm", - sha256 = "737830aab48e77ff5e6c1826f15801cfb2d68dbb622b3b39c3d7528334b73f94", - urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/vision_wasm_nosimd_internal.wasm?generation=1683564619390853"], + sha256 = "f87c51b8744b0ba564ce725fc3659dba5ef90b4615ac34135ca91c6508434fe9", + urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/vision_wasm_nosimd_internal.wasm?generation=1685638922016130"], )