diff --git a/MANIFEST.in b/MANIFEST.in index 8d5c4ec5..14afffeb 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -8,6 +8,7 @@ include README.md include requirements.txt recursive-include mediapipe/modules *.tflite *.txt *.binarypb +exclude mediapipe/modules/face_detection/face_detection_full_range.tflite exclude mediapipe/modules/objectron/object_detection_3d_chair_1stage.tflite exclude mediapipe/modules/objectron/object_detection_3d_sneakers_1stage.tflite exclude mediapipe/modules/objectron/object_detection_3d_sneakers.tflite diff --git a/README.md b/README.md index ed2fa077..8cd42fa7 100644 --- a/README.md +++ b/README.md @@ -55,46 +55,22 @@ See also [MediaPipe Models and Model Cards](https://google.github.io/mediapipe/solutions/models) for ML models released in MediaPipe. -## MediaPipe in Python - -MediaPipe offers customizable Python solutions as a prebuilt Python package on -[PyPI](https://pypi.org/project/mediapipe/), which can be installed simply with -`pip install mediapipe`. It also provides tools for users to build their own -solutions. Please see -[MediaPipe in Python](https://google.github.io/mediapipe/getting_started/python) -for more info. - -## MediaPipe on the Web - -MediaPipe on the Web is an effort to run the same ML solutions built for mobile -and desktop also in web browsers. The official API is under construction, but -the core technology has been proven effective. Please see -[MediaPipe on the Web](https://developers.googleblog.com/2020/01/mediapipe-on-web.html) -in Google Developers Blog for details. - -You can use the following links to load a demo in the MediaPipe Visualizer, and -over there click the "Runner" icon in the top bar like shown below. The demos -use your webcam video as input, which is processed all locally in real-time and -never leaves your device. - -![visualizer_runner](docs/images/visualizer_runner.png) - -* [MediaPipe Face Detection](https://viz.mediapipe.dev/demo/face_detection) -* [MediaPipe Iris](https://viz.mediapipe.dev/demo/iris_tracking) -* [MediaPipe Iris: Depth-from-Iris](https://viz.mediapipe.dev/demo/iris_depth) -* [MediaPipe Hands](https://viz.mediapipe.dev/demo/hand_tracking) -* [MediaPipe Hands (palm/hand detection only)](https://viz.mediapipe.dev/demo/hand_detection) -* [MediaPipe Pose](https://viz.mediapipe.dev/demo/pose_tracking) -* [MediaPipe Hair Segmentation](https://viz.mediapipe.dev/demo/hair_segmentation) - ## Getting started -Learn how to [install](https://google.github.io/mediapipe/getting_started/install) -MediaPipe and -[build example applications](https://google.github.io/mediapipe/getting_started/building_examples), -and start exploring our ready-to-use -[solutions](https://google.github.io/mediapipe/solutions/solutions) that you can -further extend and customize. +To start using MediaPipe +[solutions](https://google.github.io/mediapipe/solutions/solutions) with only a few +lines code, see example code and demos in +[MediaPipe in Python](https://google.github.io/mediapipe/getting_started/python) and +[MediaPipe in JavaScript](https://google.github.io/mediapipe/getting_started/javascript). + +To use MediaPipe in C++, Android and iOS, which allow further customization of +the [solutions](https://google.github.io/mediapipe/solutions/solutions) as well as +building your own, learn how to +[install](https://google.github.io/mediapipe/getting_started/install) MediaPipe and +start building example applications in +[C++](https://google.github.io/mediapipe/getting_started/cpp), +[Android](https://google.github.io/mediapipe/getting_started/android) and +[iOS](https://google.github.io/mediapipe/getting_started/ios). The source code is hosted in the [MediaPipe Github repository](https://github.com/google/mediapipe), and you can diff --git a/WORKSPACE b/WORKSPACE index 8d704eb5..15ba9589 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -351,8 +351,8 @@ maven_install( "androidx.test.espresso:espresso-core:3.1.1", "com.github.bumptech.glide:glide:4.11.0", "com.google.android.material:material:aar:1.0.0-rc01", - "com.google.auto.value:auto-value:1.6.4", - "com.google.auto.value:auto-value-annotations:1.6.4", + "com.google.auto.value:auto-value:1.8.1", + "com.google.auto.value:auto-value-annotations:1.8.1", "com.google.code.findbugs:jsr305:3.0.2", "com.google.flogger:flogger-system-backend:0.3.1", "com.google.flogger:flogger:0.3.1", diff --git a/docs/getting_started/android_archive_library.md b/docs/getting_started/android_archive_library.md index 2c2ca99f..ec34a835 100644 --- a/docs/getting_started/android_archive_library.md +++ b/docs/getting_started/android_archive_library.md @@ -92,12 +92,12 @@ each project. and copy [the binary graph](https://github.com/google/mediapipe/blob/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectiongpu/BUILD#L41) and - [the face detection tflite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_detection/face_detection_front.tflite). + [the face detection tflite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_detection/face_detection_short_range.tflite). ```bash bazel build -c opt mediapipe/graphs/face_detection:face_detection_mobile_gpu_binary_graph cp bazel-bin/mediapipe/graphs/face_detection/face_detection_mobile_gpu.binarypb /path/to/your/app/src/main/assets/ - cp mediapipe/modules/face_detection/face_detection_front.tflite /path/to/your/app/src/main/assets/ + cp mediapipe/modules/face_detection/face_detection_short_range.tflite /path/to/your/app/src/main/assets/ ``` ![Screenshot](../images/mobile/assets_location.png) @@ -117,7 +117,6 @@ each project. implementation 'com.google.flogger:flogger-system-backend:0.3.1' implementation 'com.google.code.findbugs:jsr305:3.0.2' implementation 'com.google.guava:guava:27.0.1-android' - implementation 'com.google.guava:guava:27.0.1-android' implementation 'com.google.protobuf:protobuf-java:3.11.4' // CameraX core library def camerax_version = "1.0.0-beta10" @@ -125,7 +124,7 @@ each project. implementation "androidx.camera:camera-camera2:$camerax_version" implementation "androidx.camera:camera-lifecycle:$camerax_version" // AutoValue - def auto_value_version = "1.6.4" + def auto_value_version = "1.8.1" implementation "com.google.auto.value:auto-value-annotations:$auto_value_version" annotationProcessor "com.google.auto.value:auto-value:$auto_value_version" } diff --git a/docs/images/mobile/pose_world_landmarks.mp4 b/docs/images/mobile/pose_world_landmarks.mp4 new file mode 100644 index 00000000..4a5bf301 Binary files /dev/null and b/docs/images/mobile/pose_world_landmarks.mp4 differ diff --git a/docs/index.md b/docs/index.md index cc624862..d3a22bd2 100644 --- a/docs/index.md +++ b/docs/index.md @@ -55,46 +55,22 @@ See also [MediaPipe Models and Model Cards](https://google.github.io/mediapipe/solutions/models) for ML models released in MediaPipe. -## MediaPipe in Python - -MediaPipe offers customizable Python solutions as a prebuilt Python package on -[PyPI](https://pypi.org/project/mediapipe/), which can be installed simply with -`pip install mediapipe`. It also provides tools for users to build their own -solutions. Please see -[MediaPipe in Python](https://google.github.io/mediapipe/getting_started/python) -for more info. - -## MediaPipe on the Web - -MediaPipe on the Web is an effort to run the same ML solutions built for mobile -and desktop also in web browsers. The official API is under construction, but -the core technology has been proven effective. Please see -[MediaPipe on the Web](https://developers.googleblog.com/2020/01/mediapipe-on-web.html) -in Google Developers Blog for details. - -You can use the following links to load a demo in the MediaPipe Visualizer, and -over there click the "Runner" icon in the top bar like shown below. The demos -use your webcam video as input, which is processed all locally in real-time and -never leaves your device. - -![visualizer_runner](images/visualizer_runner.png) - -* [MediaPipe Face Detection](https://viz.mediapipe.dev/demo/face_detection) -* [MediaPipe Iris](https://viz.mediapipe.dev/demo/iris_tracking) -* [MediaPipe Iris: Depth-from-Iris](https://viz.mediapipe.dev/demo/iris_depth) -* [MediaPipe Hands](https://viz.mediapipe.dev/demo/hand_tracking) -* [MediaPipe Hands (palm/hand detection only)](https://viz.mediapipe.dev/demo/hand_detection) -* [MediaPipe Pose](https://viz.mediapipe.dev/demo/pose_tracking) -* [MediaPipe Hair Segmentation](https://viz.mediapipe.dev/demo/hair_segmentation) - ## Getting started -Learn how to [install](https://google.github.io/mediapipe/getting_started/install) -MediaPipe and -[build example applications](https://google.github.io/mediapipe/getting_started/building_examples), -and start exploring our ready-to-use -[solutions](https://google.github.io/mediapipe/solutions/solutions) that you can -further extend and customize. +To start using MediaPipe +[solutions](https://google.github.io/mediapipe/solutions/solutions) with only a few +lines code, see example code and demos in +[MediaPipe in Python](https://google.github.io/mediapipe/getting_started/python) and +[MediaPipe in JavaScript](https://google.github.io/mediapipe/getting_started/javascript). + +To use MediaPipe in C++, Android and iOS, which allow further customization of +the [solutions](https://google.github.io/mediapipe/solutions/solutions) as well as +building your own, learn how to +[install](https://google.github.io/mediapipe/getting_started/install) MediaPipe and +start building example applications in +[C++](https://google.github.io/mediapipe/getting_started/cpp), +[Android](https://google.github.io/mediapipe/getting_started/android) and +[iOS](https://google.github.io/mediapipe/getting_started/ios). The source code is hosted in the [MediaPipe Github repository](https://github.com/google/mediapipe), and you can diff --git a/docs/solutions/face_detection.md b/docs/solutions/face_detection.md index e866a8cc..9d08ee48 100644 --- a/docs/solutions/face_detection.md +++ b/docs/solutions/face_detection.md @@ -45,6 +45,15 @@ section. Naming style and availability may differ slightly across platforms/languages. +#### model_selection + +An integer index `0` or `1`. Use `0` to select a short-range model that works +best for faces within 2 meters from the camera, and `1` for a full-range model +best for faces within 5 meters. For the full-range option, a sparse model is +used for its improved inference speed. Please refer to the +[model cards](./models.md#face_detection) for details. Default to `0` if not +specified. + #### min_detection_confidence Minimum confidence value (`[0.0, 1.0]`) from the face detection model for the @@ -72,6 +81,7 @@ install MediaPipe Python package, then learn more in the companion Supported configuration options: +* [model_selection](#model_selection) * [min_detection_confidence](#min_detection_confidence) ```python @@ -83,7 +93,7 @@ mp_drawing = mp.solutions.drawing_utils # For static images: IMAGE_FILES = [] with mp_face_detection.FaceDetection( - min_detection_confidence=0.5) as face_detection: + model_selection=1, min_detection_confidence=0.5) as face_detection: for idx, file in enumerate(IMAGE_FILES): image = cv2.imread(file) # Convert the BGR image to RGB and process it with MediaPipe Face Detection. @@ -103,7 +113,7 @@ with mp_face_detection.FaceDetection( # For webcam input: cap = cv2.VideoCapture(0) with mp_face_detection.FaceDetection( - min_detection_confidence=0.5) as face_detection: + model_selection=0, min_detection_confidence=0.5) as face_detection: while cap.isOpened(): success, image = cap.read() if not success: @@ -139,6 +149,7 @@ and the following usage example. Supported configuration options: +* [modelSelection](#model_selection) * [minDetectionConfidence](#min_detection_confidence) ```html @@ -189,6 +200,7 @@ const faceDetection = new FaceDetection({locateFile: (file) => { return `https://cdn.jsdelivr.net/npm/@mediapipe/face_detection@0.0/${file}`; }}); faceDetection.setOptions({ + modelSelection: 0 minDetectionConfidence: 0.5 }); faceDetection.onResults(onResults); @@ -255,10 +267,6 @@ same configuration as the GPU pipeline, runs entirely on CPU. * Target: [`mediapipe/examples/desktop/face_detection:face_detection_gpu`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/desktop/face_detection/BUILD) -### Web - -Please refer to [these instructions](../index.md#mediapipe-on-the-web). - ### Coral Please refer to diff --git a/docs/solutions/face_mesh.md b/docs/solutions/face_mesh.md index 263d9c3e..5de1b41d 100644 --- a/docs/solutions/face_mesh.md +++ b/docs/solutions/face_mesh.md @@ -69,7 +69,7 @@ and renders using a dedicated The [face landmark subgraph](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_landmark/face_landmark_front_gpu.pbtxt) internally uses a -[face_detection_subgraph](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_detection/face_detection_front_gpu.pbtxt) +[face_detection_subgraph](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_detection/face_detection_short_range_gpu.pbtxt) from the [face detection module](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_detection). diff --git a/docs/solutions/hair_segmentation.md b/docs/solutions/hair_segmentation.md index 6722c6c5..9dd997b9 100644 --- a/docs/solutions/hair_segmentation.md +++ b/docs/solutions/hair_segmentation.md @@ -51,7 +51,14 @@ to visualize its associated subgraphs, please see ### Web -Please refer to [these instructions](../index.md#mediapipe-on-the-web). +Use [this link](https://viz.mediapipe.dev/demo/hair_segmentation) to load a demo +in the MediaPipe Visualizer, and over there click the "Runner" icon in the top +bar like shown below. The demos use your webcam video as input, which is +processed all locally in real-time and never leaves your device. Please see +[MediaPipe on the Web](https://developers.googleblog.com/2020/01/mediapipe-on-web.html) +in Google Developers Blog for details. + +![visualizer_runner](../images/visualizer_runner.png) ## Resources diff --git a/docs/solutions/holistic.md b/docs/solutions/holistic.md index 14c13bd2..1ae8034b 100644 --- a/docs/solutions/holistic.md +++ b/docs/solutions/holistic.md @@ -176,6 +176,16 @@ A list of pose landmarks. Each landmark consists of the following: * `visibility`: A value in `[0.0, 1.0]` indicating the likelihood of the landmark being visible (present and not occluded) in the image. +#### pose_world_landmarks + +Another list of pose landmarks in world coordinates. Each landmark consists of +the following: + +* `x`, `y` and `z`: Real-world 3D coordinates in meters with the origin at the + center between hips. +* `visibility`: Identical to that defined in the corresponding + [pose_landmarks](#pose_landmarks). + #### face_landmarks A list of 468 face landmarks. Each landmark consists of `x`, `y` and `z`. `x` @@ -245,6 +255,9 @@ with mp_holistic.Holistic( mp_drawing.draw_landmarks( annotated_image, results.pose_landmarks, mp_holistic.POSE_CONNECTIONS) cv2.imwrite('/tmp/annotated_image' + str(idx) + '.png', annotated_image) + # Plot pose world landmarks. + mp_drawing.plot_landmarks( + results.pose_world_landmarks, mp_holistic.POSE_CONNECTIONS) # For webcam input: cap = cv2.VideoCapture(0) diff --git a/docs/solutions/iris.md b/docs/solutions/iris.md index 61ca8049..af71c895 100644 --- a/docs/solutions/iris.md +++ b/docs/solutions/iris.md @@ -69,7 +69,7 @@ and renders using a dedicated The [face landmark subgraph](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_landmark/face_landmark_front_gpu.pbtxt) internally uses a -[face detection subgraph](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_detection/face_detection_front_gpu.pbtxt) +[face detection subgraph](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_detection/face_detection_short_range_gpu.pbtxt) from the [face detection module](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_detection). @@ -193,7 +193,17 @@ on how to build MediaPipe examples. ### Web -Please refer to [these instructions](../index.md#mediapipe-on-the-web). +You can use the following links to load a demo in the MediaPipe Visualizer, and +over there click the "Runner" icon in the top bar like shown below. The demos +use your webcam video as input, which is processed all locally in real-time and +never leaves your device. Please see +[MediaPipe on the Web](https://developers.googleblog.com/2020/01/mediapipe-on-web.html) +in Google Developers Blog for details. + +![visualizer_runner](../images/visualizer_runner.png) + +* [MediaPipe Iris](https://viz.mediapipe.dev/demo/iris_tracking) +* [MediaPipe Iris: Depth-from-Iris](https://viz.mediapipe.dev/demo/iris_depth) ## Resources diff --git a/docs/solutions/models.md b/docs/solutions/models.md index 2c5e4389..2f300172 100644 --- a/docs/solutions/models.md +++ b/docs/solutions/models.md @@ -14,17 +14,27 @@ nav_order: 30 ### [Face Detection](https://google.github.io/mediapipe/solutions/face_detection) -* Face detection model for front-facing/selfie camera: - [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_detection/face_detection_front.tflite), +* Short-range model (best for faces within 2 meters from the camera): + [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_detection/face_detection_short_range.tflite), [TFLite model quantized for EdgeTPU/Coral](https://github.com/google/mediapipe/tree/master/mediapipe/examples/coral/models/face-detector-quantized_edgetpu.tflite), [Model card](https://mediapipe.page.link/blazeface-mc) -* Face detection model for back-facing camera: - [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_detection/face_detection_back.tflite), +* Full-range model (dense, best for faces within 5 meters from the camera): + [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_detection/face_detection_full_range.tflite), [Model card](https://mediapipe.page.link/blazeface-back-mc) -* Face detection model for back-facing camera (sparse): - [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_detection/face_detection_back_sparse.tflite), +* Full-range model (sparse, best for faces within 5 meters from the camera): + [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_detection/face_detection_full_range_sparse.tflite), [Model card](https://mediapipe.page.link/blazeface-back-sparse-mc) +Full-range dense and sparse models have the same quality in terms of +[F-score](https://en.wikipedia.org/wiki/F-score) however differ in underlying +metrics. The dense model is slightly better in +[Recall](https://en.wikipedia.org/wiki/Precision_and_recall) whereas the sparse +model outperforms the dense one in +[Precision](https://en.wikipedia.org/wiki/Precision_and_recall). Speed-wise +sparse model is ~30% faster when executing on CPU via +[XNNPACK](https://github.com/google/XNNPACK) whereas on GPU the models +demonstrate comparable latencies. Depending on your application, you may prefer +one over the other. ### [Face Mesh](https://google.github.io/mediapipe/solutions/face_mesh) diff --git a/docs/solutions/pose.md b/docs/solutions/pose.md index 48ce218a..0ae81a85 100644 --- a/docs/solutions/pose.md +++ b/docs/solutions/pose.md @@ -194,10 +194,23 @@ A list of pose landmarks. Each landmark consists of the following: * `z`: Represents the landmark depth with the depth at the midpoint of hips being the origin, and the smaller the value the closer the landmark is to the camera. The magnitude of `z` uses roughly the same scale as `x`. - * `visibility`: A value in `[0.0, 1.0]` indicating the likelihood of the landmark being visible (present and not occluded) in the image. +#### pose_world_landmarks + +*Fig 5. Example of MediaPipe Pose real-world 3D coordinates.* | +:-----------------------------------------------------------: | + | + +Another list of pose landmarks in world coordinates. Each landmark consists of +the following: + +* `x`, `y` and `z`: Real-world 3D coordinates in meters with the origin at the + center between hips. +* `visibility`: Identical to that defined in the corresponding + [pose_landmarks](#pose_landmarks). + ### Python Solution API Please first follow general [instructions](../getting_started/python.md) to @@ -242,6 +255,9 @@ with mp_pose.Pose( mp_drawing.draw_landmarks( annotated_image, results.pose_landmarks, mp_pose.POSE_CONNECTIONS) cv2.imwrite('/tmp/annotated_image' + str(idx) + '.png', annotated_image) + # Plot pose world landmarks. + mp_drawing.plot_landmarks( + results.pose_world_landmarks, mp_pose.POSE_CONNECTIONS) # For webcam input: cap = cv2.VideoCapture(0) @@ -294,6 +310,7 @@ Supported configuration options: + @@ -312,8 +329,15 @@ Supported configuration options: const videoElement = document.getElementsByClassName('input_video')[0]; const canvasElement = document.getElementsByClassName('output_canvas')[0]; const canvasCtx = canvasElement.getContext('2d'); +const landmarkContainer = document.getElementsByClassName('landmark-grid-container')[0]; +const grid = new LandmarkGrid(landmarkContainer); function onResults(results) { + if (!results.poseLandmarks) { + grid.updateLandmarks([]); + return; + } + canvasCtx.save(); canvasCtx.clearRect(0, 0, canvasElement.width, canvasElement.height); canvasCtx.drawImage( @@ -323,6 +347,8 @@ function onResults(results) { drawLandmarks(canvasCtx, results.poseLandmarks, {color: '#FF0000', lineWidth: 2}); canvasCtx.restore(); + + grid.updateLandmarks(results.poseWorldLandmarks); } const pose = new Pose({locateFile: (file) => { diff --git a/mediapipe/calculators/core/BUILD b/mediapipe/calculators/core/BUILD index 0c9dbcd9..1a7e7b90 100644 --- a/mediapipe/calculators/core/BUILD +++ b/mediapipe/calculators/core/BUILD @@ -933,8 +933,8 @@ cc_test( ) cc_library( - name = "split_normalized_landmark_list_calculator", - srcs = ["split_normalized_landmark_list_calculator.cc"], + name = "split_landmarks_calculator", + srcs = ["split_landmarks_calculator.cc"], visibility = ["//visibility:public"], deps = [ ":split_vector_calculator_cc_proto", @@ -948,10 +948,10 @@ cc_library( ) cc_test( - name = "split_normalized_landmark_list_calculator_test", - srcs = ["split_normalized_landmark_list_calculator_test.cc"], + name = "split_landmarks_calculator_test", + srcs = ["split_landmarks_calculator_test.cc"], deps = [ - ":split_normalized_landmark_list_calculator", + ":split_landmarks_calculator", ":split_vector_calculator_cc_proto", "//mediapipe/framework:calculator_framework", "//mediapipe/framework:calculator_runner", diff --git a/mediapipe/calculators/core/split_normalized_landmark_list_calculator.cc b/mediapipe/calculators/core/split_landmarks_calculator.cc similarity index 75% rename from mediapipe/calculators/core/split_normalized_landmark_list_calculator.cc rename to mediapipe/calculators/core/split_landmarks_calculator.cc index d57cebe9..5bc876bf 100644 --- a/mediapipe/calculators/core/split_normalized_landmark_list_calculator.cc +++ b/mediapipe/calculators/core/split_landmarks_calculator.cc @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef MEDIAPIPE_CALCULATORS_CORE_SPLIT_NORMALIZED_LANDMARK_LIST_CALCULATOR_H_ // NOLINT -#define MEDIAPIPE_CALCULATORS_CORE_SPLIT_NORMALIZED_LANDMARK_LIST_CALCULATOR_H_ // NOLINT +#ifndef MEDIAPIPE_CALCULATORS_CORE_SPLIT_LANDMARKS_CALCULATOR_H_ // NOLINT +#define MEDIAPIPE_CALCULATORS_CORE_SPLIT_LANDMARKS_CALCULATOR_H_ // NOLINT #include "mediapipe/calculators/core/split_vector_calculator.pb.h" #include "mediapipe/framework/calculator_framework.h" @@ -24,29 +24,30 @@ namespace mediapipe { -// Splits an input packet with NormalizedLandmarkList into -// multiple NormalizedLandmarkList output packets using the [begin, end) ranges +// Splits an input packet with LandmarkListType into +// multiple LandmarkListType output packets using the [begin, end) ranges // specified in SplitVectorCalculatorOptions. If the option "element_only" is // set to true, all ranges should be of size 1 and all outputs will be elements -// of type NormalizedLandmark. If "element_only" is false, ranges can be -// non-zero in size and all outputs will be of type NormalizedLandmarkList. +// of type LandmarkType. If "element_only" is false, ranges can be +// non-zero in size and all outputs will be of type LandmarkListType. // If the option "combine_outputs" is set to true, only one output stream can be // specified and all ranges of elements will be combined into one -// NormalizedLandmarkList. -class SplitNormalizedLandmarkListCalculator : public CalculatorBase { +// LandmarkListType. +template +class SplitLandmarksCalculator : public CalculatorBase { public: static absl::Status GetContract(CalculatorContract* cc) { RET_CHECK(cc->Inputs().NumEntries() == 1); RET_CHECK(cc->Outputs().NumEntries() != 0); - cc->Inputs().Index(0).Set(); + cc->Inputs().Index(0).Set(); const auto& options = cc->Options<::mediapipe::SplitVectorCalculatorOptions>(); if (options.combine_outputs()) { RET_CHECK_EQ(cc->Outputs().NumEntries(), 1); - cc->Outputs().Index(0).Set(); + cc->Outputs().Index(0).Set(); for (int i = 0; i < options.ranges_size() - 1; ++i) { for (int j = i + 1; j < options.ranges_size(); ++j) { const auto& range_0 = options.ranges(i); @@ -81,9 +82,9 @@ class SplitNormalizedLandmarkListCalculator : public CalculatorBase { return absl::InvalidArgumentError( "Since element_only is true, all ranges should be of size 1."); } - cc->Outputs().Index(i).Set(); + cc->Outputs().Index(i).Set(); } else { - cc->Outputs().Index(i).Set(); + cc->Outputs().Index(i).Set(); } } } @@ -110,40 +111,39 @@ class SplitNormalizedLandmarkListCalculator : public CalculatorBase { } absl::Status Process(CalculatorContext* cc) override { - const NormalizedLandmarkList& input = - cc->Inputs().Index(0).Get(); + const LandmarkListType& input = + cc->Inputs().Index(0).Get(); RET_CHECK_GE(input.landmark_size(), max_range_end_) << "Max range end " << max_range_end_ << " exceeds landmarks size " << input.landmark_size(); if (combine_outputs_) { - NormalizedLandmarkList output; + LandmarkListType output; for (int i = 0; i < ranges_.size(); ++i) { for (int j = ranges_[i].first; j < ranges_[i].second; ++j) { - const NormalizedLandmark& input_landmark = input.landmark(j); + const LandmarkType& input_landmark = input.landmark(j); *output.add_landmark() = input_landmark; } } RET_CHECK_EQ(output.landmark_size(), total_elements_); cc->Outputs().Index(0).AddPacket( - MakePacket(output).At(cc->InputTimestamp())); + MakePacket(output).At(cc->InputTimestamp())); } else { if (element_only_) { for (int i = 0; i < ranges_.size(); ++i) { cc->Outputs().Index(i).AddPacket( - MakePacket(input.landmark(ranges_[i].first)) + MakePacket(input.landmark(ranges_[i].first)) .At(cc->InputTimestamp())); } } else { for (int i = 0; i < ranges_.size(); ++i) { - NormalizedLandmarkList output; + LandmarkListType output; for (int j = ranges_[i].first; j < ranges_[i].second; ++j) { - const NormalizedLandmark& input_landmark = input.landmark(j); + const LandmarkType& input_landmark = input.landmark(j); *output.add_landmark() = input_landmark; } cc->Outputs().Index(i).AddPacket( - MakePacket(output).At( - cc->InputTimestamp())); + MakePacket(output).At(cc->InputTimestamp())); } } } @@ -159,9 +159,15 @@ class SplitNormalizedLandmarkListCalculator : public CalculatorBase { bool combine_outputs_ = false; }; +typedef SplitLandmarksCalculator + SplitNormalizedLandmarkListCalculator; REGISTER_CALCULATOR(SplitNormalizedLandmarkListCalculator); +typedef SplitLandmarksCalculator + SplitLandmarkListCalculator; +REGISTER_CALCULATOR(SplitLandmarkListCalculator); + } // namespace mediapipe // NOLINTNEXTLINE -#endif // MEDIAPIPE_CALCULATORS_CORE_SPLIT_NORMALIZED_LANDMARK_LIST_CALCULATOR_H_ +#endif // MEDIAPIPE_CALCULATORS_CORE_SPLIT_LANDMARKS_CALCULATOR_H_ diff --git a/mediapipe/calculators/core/split_normalized_landmark_list_calculator_test.cc b/mediapipe/calculators/core/split_landmarks_calculator_test.cc similarity index 100% rename from mediapipe/calculators/core/split_normalized_landmark_list_calculator_test.cc rename to mediapipe/calculators/core/split_landmarks_calculator_test.cc diff --git a/mediapipe/calculators/image/BUILD b/mediapipe/calculators/image/BUILD index 39f81c04..507b6f0f 100644 --- a/mediapipe/calculators/image/BUILD +++ b/mediapipe/calculators/image/BUILD @@ -80,6 +80,16 @@ mediapipe_proto_library( ], ) +mediapipe_proto_library( + name = "segmentation_smoothing_calculator_proto", + srcs = ["segmentation_smoothing_calculator.proto"], + visibility = ["//visibility:public"], + deps = [ + "//mediapipe/framework:calculator_options_proto", + "//mediapipe/framework:calculator_proto", + ], +) + cc_library( name = "color_convert_calculator", srcs = ["color_convert_calculator.cc"], @@ -602,3 +612,52 @@ cc_test( "//mediapipe/framework/port:parse_text_proto", ], ) + +cc_library( + name = "segmentation_smoothing_calculator", + srcs = ["segmentation_smoothing_calculator.cc"], + visibility = ["//visibility:public"], + deps = [ + ":segmentation_smoothing_calculator_cc_proto", + "//mediapipe/framework:calculator_options_cc_proto", + "//mediapipe/framework/formats:image_format_cc_proto", + "//mediapipe/framework:calculator_framework", + "//mediapipe/framework/formats:image_frame", + "//mediapipe/framework/formats:image_frame_opencv", + "//mediapipe/framework/formats:image", + "//mediapipe/framework/formats:image_opencv", + "//mediapipe/framework/port:logging", + "//mediapipe/framework/port:opencv_core", + "//mediapipe/framework/port:status", + "//mediapipe/framework/port:vector", + ] + select({ + "//mediapipe/gpu:disable_gpu": [], + "//conditions:default": [ + "//mediapipe/gpu:gl_calculator_helper", + "//mediapipe/gpu:gl_simple_shaders", + "//mediapipe/gpu:gl_quad_renderer", + "//mediapipe/gpu:shader_util", + ], + }), + alwayslink = 1, +) + +cc_test( + name = "segmentation_smoothing_calculator_test", + srcs = ["segmentation_smoothing_calculator_test.cc"], + deps = [ + ":image_clone_calculator", + ":image_clone_calculator_cc_proto", + ":segmentation_smoothing_calculator", + ":segmentation_smoothing_calculator_cc_proto", + "//mediapipe/framework:calculator_framework", + "//mediapipe/framework:calculator_runner", + "//mediapipe/framework/deps:file_path", + "//mediapipe/framework/formats:image_frame", + "//mediapipe/framework/formats:image_opencv", + "//mediapipe/framework/port:gtest_main", + "//mediapipe/framework/port:opencv_imgcodecs", + "//mediapipe/framework/port:opencv_imgproc", + "//mediapipe/framework/port:parse_text_proto", + ], +) diff --git a/mediapipe/calculators/image/segmentation_smoothing_calculator.cc b/mediapipe/calculators/image/segmentation_smoothing_calculator.cc new file mode 100644 index 00000000..db339b75 --- /dev/null +++ b/mediapipe/calculators/image/segmentation_smoothing_calculator.cc @@ -0,0 +1,429 @@ +// Copyright 2021 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include + +#include "mediapipe/calculators/image/segmentation_smoothing_calculator.pb.h" +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/calculator_options.pb.h" +#include "mediapipe/framework/formats/image.h" +#include "mediapipe/framework/formats/image_format.pb.h" +#include "mediapipe/framework/formats/image_frame.h" +#include "mediapipe/framework/formats/image_frame_opencv.h" +#include "mediapipe/framework/formats/image_opencv.h" +#include "mediapipe/framework/port/logging.h" +#include "mediapipe/framework/port/opencv_core_inc.h" +#include "mediapipe/framework/port/status.h" +#include "mediapipe/framework/port/vector.h" + +#if !MEDIAPIPE_DISABLE_GPU +#include "mediapipe/gpu/gl_calculator_helper.h" +#include "mediapipe/gpu/gl_simple_shaders.h" +#include "mediapipe/gpu/shader_util.h" +#endif // !MEDIAPIPE_DISABLE_GPU + +namespace mediapipe { + +namespace { +constexpr char kCurrentMaskTag[] = "MASK"; +constexpr char kPreviousMaskTag[] = "MASK_PREVIOUS"; +constexpr char kOutputMaskTag[] = "MASK_SMOOTHED"; + +enum { ATTRIB_VERTEX, ATTRIB_TEXTURE_POSITION, NUM_ATTRIBUTES }; +} // namespace + +// A calculator for mixing two segmentation masks together, +// based on an uncertantity probability estimate. +// +// Inputs: +// MASK - Image containing the new/current mask. +// [ImageFormat::VEC32F1, or +// GpuBufferFormat::kBGRA32/kRGB24/kGrayHalf16/kGrayFloat32] +// MASK_PREVIOUS - Image containing previous mask. +// [Same format as MASK_CURRENT] +// * If input channels is >1, only the first channel (R) is used as the mask. +// +// Output: +// MASK_SMOOTHED - Blended mask. +// [Same format as MASK_CURRENT] +// * The resulting filtered mask will be stored in R channel, +// and duplicated in A if 4 channels. +// +// Options: +// combine_with_previous_ratio - Amount of previous to blend with current. +// +// Example: +// node { +// calculator: "SegmentationSmoothingCalculator" +// input_stream: "MASK:mask" +// input_stream: "MASK_PREVIOUS:mask_previous" +// output_stream: "MASK_SMOOTHED:mask_smoothed" +// options: { +// [mediapipe.SegmentationSmoothingCalculatorOptions.ext] { +// combine_with_previous_ratio: 0.9 +// } +// } +// } +// +class SegmentationSmoothingCalculator : public CalculatorBase { + public: + SegmentationSmoothingCalculator() = default; + + static absl::Status GetContract(CalculatorContract* cc); + + // From Calculator. + absl::Status Open(CalculatorContext* cc) override; + absl::Status Process(CalculatorContext* cc) override; + absl::Status Close(CalculatorContext* cc) override; + + private: + absl::Status RenderGpu(CalculatorContext* cc); + absl::Status RenderCpu(CalculatorContext* cc); + + absl::Status GlSetup(CalculatorContext* cc); + void GlRender(CalculatorContext* cc); + + float combine_with_previous_ratio_; + + bool gpu_initialized_ = false; +#if !MEDIAPIPE_DISABLE_GPU + mediapipe::GlCalculatorHelper gpu_helper_; + GLuint program_ = 0; +#endif // !MEDIAPIPE_DISABLE_GPU +}; +REGISTER_CALCULATOR(SegmentationSmoothingCalculator); + +absl::Status SegmentationSmoothingCalculator::GetContract( + CalculatorContract* cc) { + CHECK_GE(cc->Inputs().NumEntries(), 1); + + cc->Inputs().Tag(kCurrentMaskTag).Set(); + cc->Inputs().Tag(kPreviousMaskTag).Set(); + cc->Outputs().Tag(kOutputMaskTag).Set(); + +#if !MEDIAPIPE_DISABLE_GPU + MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc)); +#endif // !MEDIAPIPE_DISABLE_GPU + + return absl::OkStatus(); +} + +absl::Status SegmentationSmoothingCalculator::Open(CalculatorContext* cc) { + cc->SetOffset(TimestampDiff(0)); + + auto options = + cc->Options(); + combine_with_previous_ratio_ = options.combine_with_previous_ratio(); + +#if !MEDIAPIPE_DISABLE_GPU + MP_RETURN_IF_ERROR(gpu_helper_.Open(cc)); +#endif // !MEDIAPIPE_DISABLE_GPU + + return absl::OkStatus(); +} + +absl::Status SegmentationSmoothingCalculator::Process(CalculatorContext* cc) { + if (cc->Inputs().Tag(kCurrentMaskTag).IsEmpty()) { + return absl::OkStatus(); + } + if (cc->Inputs().Tag(kPreviousMaskTag).IsEmpty()) { + // Pass through current image if previous is not available. + cc->Outputs() + .Tag(kOutputMaskTag) + .AddPacket(cc->Inputs().Tag(kCurrentMaskTag).Value()); + return absl::OkStatus(); + } + + // Run on GPU if incoming data is on GPU. + const bool use_gpu = cc->Inputs().Tag(kCurrentMaskTag).Get().UsesGpu(); + + if (use_gpu) { +#if !MEDIAPIPE_DISABLE_GPU + MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext([this, cc]() -> absl::Status { + if (!gpu_initialized_) { + MP_RETURN_IF_ERROR(GlSetup(cc)); + gpu_initialized_ = true; + } + MP_RETURN_IF_ERROR(RenderGpu(cc)); + return absl::OkStatus(); + })); +#else + return absl::InternalError("GPU processing is disabled."); +#endif // !MEDIAPIPE_DISABLE_GPU + } else { + MP_RETURN_IF_ERROR(RenderCpu(cc)); + } + + return absl::OkStatus(); +} + +absl::Status SegmentationSmoothingCalculator::Close(CalculatorContext* cc) { +#if !MEDIAPIPE_DISABLE_GPU + gpu_helper_.RunInGlContext([this] { + if (program_) glDeleteProgram(program_); + program_ = 0; + }); +#endif // !MEDIAPIPE_DISABLE_GPU + + return absl::OkStatus(); +} + +absl::Status SegmentationSmoothingCalculator::RenderCpu(CalculatorContext* cc) { + // Setup source images. + const auto& current_frame = cc->Inputs().Tag(kCurrentMaskTag).Get(); + const cv::Mat current_mat = mediapipe::formats::MatView(¤t_frame); + RET_CHECK_EQ(current_mat.type(), CV_32FC1) + << "Only 1-channel float input image is supported."; + + const auto& previous_frame = cc->Inputs().Tag(kPreviousMaskTag).Get(); + const cv::Mat previous_mat = mediapipe::formats::MatView(&previous_frame); + RET_CHECK_EQ(previous_mat.type(), current_mat.type()) + << "Warning: mixing input format types: " << previous_mat.type() + << " != " << previous_mat.type(); + + RET_CHECK_EQ(current_mat.rows, previous_mat.rows); + RET_CHECK_EQ(current_mat.cols, previous_mat.cols); + + // Setup destination image. + auto output_frame = std::make_shared( + current_frame.image_format(), current_mat.cols, current_mat.rows); + cv::Mat output_mat = mediapipe::formats::MatView(output_frame.get()); + output_mat.setTo(cv::Scalar(0)); + + // Blending function. + const auto blending_fn = [&](const float prev_mask_value, + const float new_mask_value) { + /* + * Assume p := new_mask_value + * H(p) := 1 + (p * log(p) + (1-p) * log(1-p)) / log(2) + * uncertainty alpha(p) = + * Clamp(1 - (1 - H(p)) * (1 - H(p)), 0, 1) [squaring the uncertainty] + * + * The following polynomial approximates uncertainty alpha as a function + * of (p + 0.5): + */ + const float c1 = 5.68842; + const float c2 = -0.748699; + const float c3 = -57.8051; + const float c4 = 291.309; + const float c5 = -624.717; + const float t = new_mask_value - 0.5f; + const float x = t * t; + + const float uncertainty = + 1.0f - + std::min(1.0f, x * (c1 + x * (c2 + x * (c3 + x * (c4 + x * c5))))); + + return new_mask_value + (prev_mask_value - new_mask_value) * + (uncertainty * combine_with_previous_ratio_); + }; + + // Write directly to the first channel of output. + for (int i = 0; i < output_mat.rows; ++i) { + float* out_ptr = output_mat.ptr(i); + const float* curr_ptr = current_mat.ptr(i); + const float* prev_ptr = previous_mat.ptr(i); + for (int j = 0; j < output_mat.cols; ++j) { + const float new_mask_value = curr_ptr[j]; + const float prev_mask_value = prev_ptr[j]; + out_ptr[j] = blending_fn(prev_mask_value, new_mask_value); + } + } + + cc->Outputs() + .Tag(kOutputMaskTag) + .AddPacket(MakePacket(output_frame).At(cc->InputTimestamp())); + + return absl::OkStatus(); +} + +absl::Status SegmentationSmoothingCalculator::RenderGpu(CalculatorContext* cc) { +#if !MEDIAPIPE_DISABLE_GPU + // Setup source textures. + const auto& current_frame = cc->Inputs().Tag(kCurrentMaskTag).Get(); + RET_CHECK( + (current_frame.format() == mediapipe::GpuBufferFormat::kBGRA32 || + current_frame.format() == mediapipe::GpuBufferFormat::kGrayHalf16 || + current_frame.format() == mediapipe::GpuBufferFormat::kGrayFloat32 || + current_frame.format() == mediapipe::GpuBufferFormat::kRGB24)) + << "Only RGBA, RGB, or 1-channel Float input image supported."; + + auto current_texture = gpu_helper_.CreateSourceTexture(current_frame); + + const auto& previous_frame = cc->Inputs().Tag(kPreviousMaskTag).Get(); + if (previous_frame.format() != current_frame.format()) { + LOG(ERROR) << "Warning: mixing input format types. "; + } + auto previous_texture = gpu_helper_.CreateSourceTexture(previous_frame); + + // Setup destination texture. + const int width = current_frame.width(), height = current_frame.height(); + auto output_texture = gpu_helper_.CreateDestinationTexture( + width, height, current_frame.format()); + + // Process shader. + { + gpu_helper_.BindFramebuffer(output_texture); + glActiveTexture(GL_TEXTURE1); + glBindTexture(GL_TEXTURE_2D, current_texture.name()); + glActiveTexture(GL_TEXTURE2); + glBindTexture(GL_TEXTURE_2D, previous_texture.name()); + GlRender(cc); + glActiveTexture(GL_TEXTURE2); + glBindTexture(GL_TEXTURE_2D, 0); + glActiveTexture(GL_TEXTURE1); + glBindTexture(GL_TEXTURE_2D, 0); + } + glFlush(); + + // Send out image as GPU packet. + auto output_frame = output_texture.GetFrame(); + cc->Outputs() + .Tag(kOutputMaskTag) + .Add(output_frame.release(), cc->InputTimestamp()); +#endif // !MEDIAPIPE_DISABLE_GPU + + return absl::OkStatus(); +} + +void SegmentationSmoothingCalculator::GlRender(CalculatorContext* cc) { +#if !MEDIAPIPE_DISABLE_GPU + static const GLfloat square_vertices[] = { + -1.0f, -1.0f, // bottom left + 1.0f, -1.0f, // bottom right + -1.0f, 1.0f, // top left + 1.0f, 1.0f, // top right + }; + static const GLfloat texture_vertices[] = { + 0.0f, 0.0f, // bottom left + 1.0f, 0.0f, // bottom right + 0.0f, 1.0f, // top left + 1.0f, 1.0f, // top right + }; + + // program + glUseProgram(program_); + + // vertex storage + GLuint vbo[2]; + glGenBuffers(2, vbo); + GLuint vao; + glGenVertexArrays(1, &vao); + glBindVertexArray(vao); + + // vbo 0 + glBindBuffer(GL_ARRAY_BUFFER, vbo[0]); + glBufferData(GL_ARRAY_BUFFER, 4 * 2 * sizeof(GLfloat), square_vertices, + GL_STATIC_DRAW); + glEnableVertexAttribArray(ATTRIB_VERTEX); + glVertexAttribPointer(ATTRIB_VERTEX, 2, GL_FLOAT, 0, 0, nullptr); + + // vbo 1 + glBindBuffer(GL_ARRAY_BUFFER, vbo[1]); + glBufferData(GL_ARRAY_BUFFER, 4 * 2 * sizeof(GLfloat), texture_vertices, + GL_STATIC_DRAW); + glEnableVertexAttribArray(ATTRIB_TEXTURE_POSITION); + glVertexAttribPointer(ATTRIB_TEXTURE_POSITION, 2, GL_FLOAT, 0, 0, nullptr); + + // draw + glDrawArrays(GL_TRIANGLE_STRIP, 0, 4); + + // cleanup + glDisableVertexAttribArray(ATTRIB_VERTEX); + glDisableVertexAttribArray(ATTRIB_TEXTURE_POSITION); + glBindBuffer(GL_ARRAY_BUFFER, 0); + glBindVertexArray(0); + glDeleteVertexArrays(1, &vao); + glDeleteBuffers(2, vbo); + +#endif // !MEDIAPIPE_DISABLE_GPU +} + +absl::Status SegmentationSmoothingCalculator::GlSetup(CalculatorContext* cc) { +#if !MEDIAPIPE_DISABLE_GPU + const GLint attr_location[NUM_ATTRIBUTES] = { + ATTRIB_VERTEX, + ATTRIB_TEXTURE_POSITION, + }; + const GLchar* attr_name[NUM_ATTRIBUTES] = { + "position", + "texture_coordinate", + }; + + // Shader to blend in previous mask based on computed uncertainty probability. + const std::string frag_src = + absl::StrCat(std::string(mediapipe::kMediaPipeFragmentShaderPreamble), + R"( + DEFAULT_PRECISION(mediump, float) + + #ifdef GL_ES + #define fragColor gl_FragColor + #else + out vec4 fragColor; + #endif // defined(GL_ES); + + in vec2 sample_coordinate; + uniform sampler2D current_mask; + uniform sampler2D previous_mask; + uniform float combine_with_previous_ratio; + + void main() { + vec4 current_pix = texture2D(current_mask, sample_coordinate); + vec4 previous_pix = texture2D(previous_mask, sample_coordinate); + float new_mask_value = current_pix.r; + float prev_mask_value = previous_pix.r; + + // Assume p := new_mask_value + // H(p) := 1 + (p * log(p) + (1-p) * log(1-p)) / log(2) + // uncertainty alpha(p) = + // Clamp(1 - (1 - H(p)) * (1 - H(p)), 0, 1) [squaring the uncertainty] + // + // The following polynomial approximates uncertainty alpha as a function + // of (p + 0.5): + const float c1 = 5.68842; + const float c2 = -0.748699; + const float c3 = -57.8051; + const float c4 = 291.309; + const float c5 = -624.717; + float t = new_mask_value - 0.5; + float x = t * t; + + float uncertainty = + 1.0 - min(1.0, x * (c1 + x * (c2 + x * (c3 + x * (c4 + x * c5))))); + + new_mask_value += + (prev_mask_value - new_mask_value) * (uncertainty * combine_with_previous_ratio); + + fragColor = vec4(new_mask_value, 0.0, 0.0, new_mask_value); + } + )"); + + // Create shader program and set parameters. + mediapipe::GlhCreateProgram(mediapipe::kBasicVertexShader, frag_src.c_str(), + NUM_ATTRIBUTES, (const GLchar**)&attr_name[0], + attr_location, &program_); + RET_CHECK(program_) << "Problem initializing the program."; + glUseProgram(program_); + glUniform1i(glGetUniformLocation(program_, "current_mask"), 1); + glUniform1i(glGetUniformLocation(program_, "previous_mask"), 2); + glUniform1f(glGetUniformLocation(program_, "combine_with_previous_ratio"), + combine_with_previous_ratio_); + +#endif // !MEDIAPIPE_DISABLE_GPU + + return absl::OkStatus(); +} + +} // namespace mediapipe diff --git a/mediapipe/calculators/image/segmentation_smoothing_calculator.proto b/mediapipe/calculators/image/segmentation_smoothing_calculator.proto new file mode 100644 index 00000000..12b10ccd --- /dev/null +++ b/mediapipe/calculators/image/segmentation_smoothing_calculator.proto @@ -0,0 +1,35 @@ +// Copyright 2021 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto2"; + +package mediapipe; + +import "mediapipe/framework/calculator.proto"; + +message SegmentationSmoothingCalculatorOptions { + extend CalculatorOptions { + optional SegmentationSmoothingCalculatorOptions ext = 377425128; + } + + // How much to blend in previous mask, based on a probability estimate. + // Range: [0-1] + // 0 = Use only current frame (no blending). + // 1 = Blend in the previous mask based on uncertainty estimate. + // With ratio at 1, the uncertainty estimate is trusted completely. + // When uncertainty is high, the previous mask is given higher weight. + // Therefore, if both ratio and uncertainty are 1, only old mask is used. + // A pixel is 'uncertain' if its value is close to the middle (0.5 or 127). + optional float combine_with_previous_ratio = 1 [default = 0.0]; +} diff --git a/mediapipe/calculators/image/segmentation_smoothing_calculator_test.cc b/mediapipe/calculators/image/segmentation_smoothing_calculator_test.cc new file mode 100644 index 00000000..100d7de8 --- /dev/null +++ b/mediapipe/calculators/image/segmentation_smoothing_calculator_test.cc @@ -0,0 +1,206 @@ +// Copyright 2018 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include "mediapipe/calculators/image/segmentation_smoothing_calculator.pb.h" +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/calculator_runner.h" +#include "mediapipe/framework/deps/file_path.h" +#include "mediapipe/framework/formats/image_frame.h" +#include "mediapipe/framework/formats/image_opencv.h" +#include "mediapipe/framework/port/gmock.h" +#include "mediapipe/framework/port/gtest.h" +#include "mediapipe/framework/port/opencv_imgcodecs_inc.h" +#include "mediapipe/framework/port/opencv_imgproc_inc.h" +#include "mediapipe/framework/port/parse_text_proto.h" +#include "mediapipe/framework/port/status_matchers.h" + +namespace mediapipe { + +namespace { + +// 4x4 VEC32F1, center 2x2 block set at ~250 +const float mask_data[] = { + 0.00, 0.00, 0.00, 0.00, // + 0.00, 0.98, 0.98, 0.00, // + 0.00, 0.98, 0.98, 0.00, // + 0.00, 0.00, 0.00, 0.00, // +}; + +void RunGraph(Packet curr_packet, Packet prev_packet, bool use_gpu, float ratio, + cv::Mat* result) { + CalculatorGraphConfig graph_config; + if (use_gpu) { + graph_config = ParseTextProtoOrDie(absl::Substitute( + R"pb( + input_stream: "curr_mask" + input_stream: "prev_mask" + output_stream: "new_mask" + node { + calculator: "ImageCloneCalculator" + input_stream: "curr_mask" + output_stream: "curr_mask_gpu" + options: { + [mediapipe.ImageCloneCalculatorOptions.ext] { + output_on_gpu: true + } + } + } + node { + calculator: "ImageCloneCalculator" + input_stream: "prev_mask" + output_stream: "prev_mask_gpu" + options: { + [mediapipe.ImageCloneCalculatorOptions.ext] { + output_on_gpu: true + } + } + } + node { + calculator: "SegmentationSmoothingCalculator" + input_stream: "MASK:curr_mask_gpu" + input_stream: "MASK_PREVIOUS:prev_mask_gpu" + output_stream: "MASK_SMOOTHED:new_mask" + node_options { + [type.googleapis.com/ + mediapipe.SegmentationSmoothingCalculatorOptions]: { + combine_with_previous_ratio: $0 + } + } + } + )pb", + ratio)); + } else { + graph_config = ParseTextProtoOrDie(absl::Substitute( + R"pb( + input_stream: "curr_mask" + input_stream: "prev_mask" + output_stream: "new_mask" + node { + calculator: "SegmentationSmoothingCalculator" + input_stream: "MASK:curr_mask" + input_stream: "MASK_PREVIOUS:prev_mask" + output_stream: "MASK_SMOOTHED:new_mask" + node_options { + [type.googleapis.com/ + mediapipe.SegmentationSmoothingCalculatorOptions]: { + combine_with_previous_ratio: $0 + } + } + } + )pb", + ratio)); + } + std::vector output_packets; + tool::AddVectorSink("new_mask", &graph_config, &output_packets); + CalculatorGraph graph(graph_config); + MP_ASSERT_OK(graph.StartRun({})); + + MP_ASSERT_OK( + graph.AddPacketToInputStream("curr_mask", curr_packet.At(Timestamp(0)))); + MP_ASSERT_OK( + graph.AddPacketToInputStream("prev_mask", prev_packet.At(Timestamp(0)))); + MP_ASSERT_OK(graph.WaitUntilIdle()); + ASSERT_EQ(1, output_packets.size()); + + Image result_image = output_packets[0].Get(); + cv::Mat result_mat = formats::MatView(&result_image); + result_mat.copyTo(*result); + + // Fully close graph at end, otherwise calculator+Images are destroyed + // after calling WaitUntilDone(). + MP_ASSERT_OK(graph.CloseInputStream("curr_mask")); + MP_ASSERT_OK(graph.CloseInputStream("prev_mask")); + MP_ASSERT_OK(graph.WaitUntilDone()); +} + +void RunTest(bool use_gpu, float mix_ratio, cv::Mat& test_result) { + cv::Mat mask_mat(cv::Size(4, 4), CV_32FC1, const_cast(mask_data)); + cv::Mat curr_mat = mask_mat; + // 3x3 blur of 250 block produces all pixels '111'. + cv::Mat prev_mat; + cv::blur(mask_mat, prev_mat, cv::Size(3, 3)); + + Packet curr_packet = MakePacket(std::make_unique( + ImageFormat::VEC32F1, curr_mat.size().width, curr_mat.size().height)); + curr_mat.copyTo(formats::MatView(&(curr_packet.Get()))); + Packet prev_packet = MakePacket(std::make_unique( + ImageFormat::VEC32F1, prev_mat.size().width, prev_mat.size().height)); + prev_mat.copyTo(formats::MatView(&(prev_packet.Get()))); + + cv::Mat result; + RunGraph(curr_packet, prev_packet, use_gpu, mix_ratio, &result); + + ASSERT_EQ(curr_mat.rows, result.rows); + ASSERT_EQ(curr_mat.cols, result.cols); + ASSERT_EQ(curr_mat.type(), result.type()); + result.copyTo(test_result); + + if (mix_ratio == 1.0) { + for (int i = 0; i < 4; ++i) { + for (int j = 0; j < 4; ++j) { + float in = curr_mat.at(i, j); + float out = result.at(i, j); + // Since the input has high value (250), it has low uncertainty. + // So the output should have changed lower (towards prev), + // but not too much. + if (in > 0) EXPECT_NE(in, out); + EXPECT_NEAR(in, out, 3.0 / 255.0); + } + } + } else if (mix_ratio == 0.0) { + for (int i = 0; i < 4; ++i) { + for (int j = 0; j < 4; ++j) { + float in = curr_mat.at(i, j); + float out = result.at(i, j); + EXPECT_EQ(in, out); // Output should match current. + } + } + } else { + LOG(ERROR) << "invalid ratio"; + } +} + +TEST(SegmentationSmoothingCalculatorTest, TestSmoothing) { + bool use_gpu; + float mix_ratio; + + use_gpu = false; + mix_ratio = 0.0; + cv::Mat cpu_0; + RunTest(use_gpu, mix_ratio, cpu_0); + + use_gpu = false; + mix_ratio = 1.0; + cv::Mat cpu_1; + RunTest(use_gpu, mix_ratio, cpu_1); + + use_gpu = true; + mix_ratio = 1.0; + cv::Mat gpu_1; + RunTest(use_gpu, mix_ratio, gpu_1); + + // CPU & GPU should match. + for (int i = 0; i < 4; ++i) { + for (int j = 0; j < 4; ++j) { + float gpu = gpu_1.at(i, j); + float cpu = cpu_1.at(i, j); + EXPECT_EQ(cpu, gpu); + } + } +} + +} // namespace +} // namespace mediapipe diff --git a/mediapipe/calculators/tensor/BUILD b/mediapipe/calculators/tensor/BUILD index 3979def5..71be05f6 100644 --- a/mediapipe/calculators/tensor/BUILD +++ b/mediapipe/calculators/tensor/BUILD @@ -109,6 +109,8 @@ cc_library( "//mediapipe/gpu:MPPMetalUtil", "//mediapipe/gpu:gpu_buffer", "//mediapipe/objc:mediapipe_framework_ios", + "//mediapipe/util/tflite:config", + "@com_google_absl//absl/memory", "@org_tensorflow//tensorflow/lite/delegates/gpu:metal_delegate", "@org_tensorflow//tensorflow/lite/delegates/gpu:metal_delegate_internal", "@org_tensorflow//tensorflow/lite/delegates/gpu/common:shape", @@ -478,7 +480,6 @@ cc_library( deps = [ ":image_to_tensor_calculator_cc_proto", ":image_to_tensor_converter", - ":image_to_tensor_converter_opencv", ":image_to_tensor_utils", "//mediapipe/framework/api2:node", "//mediapipe/framework/formats:image", @@ -494,6 +495,9 @@ cc_library( ] + select({ "//mediapipe/gpu:disable_gpu": [], "//conditions:default": [":image_to_tensor_calculator_gpu_deps"], + }) + select({ + "//mediapipe/framework/port:disable_opencv": [], + "//conditions:default": [":image_to_tensor_converter_opencv"], }), alwayslink = 1, ) diff --git a/mediapipe/calculators/tensor/image_to_tensor_calculator.cc b/mediapipe/calculators/tensor/image_to_tensor_calculator.cc index f681ab66..5c22d734 100644 --- a/mediapipe/calculators/tensor/image_to_tensor_calculator.cc +++ b/mediapipe/calculators/tensor/image_to_tensor_calculator.cc @@ -18,7 +18,6 @@ #include "mediapipe/calculators/tensor/image_to_tensor_calculator.pb.h" #include "mediapipe/calculators/tensor/image_to_tensor_converter.h" -#include "mediapipe/calculators/tensor/image_to_tensor_converter_opencv.h" #include "mediapipe/calculators/tensor/image_to_tensor_utils.h" #include "mediapipe/framework/api2/node.h" #include "mediapipe/framework/calculator_framework.h" @@ -33,6 +32,10 @@ #include "mediapipe/framework/port/statusor.h" #include "mediapipe/gpu/gpu_origin.pb.h" +#if !MEDIAPIPE_DISABLE_OPENCV +#include "mediapipe/calculators/tensor/image_to_tensor_converter_opencv.h" +#endif + #if !MEDIAPIPE_DISABLE_GPU #include "mediapipe/gpu/gpu_buffer.h" @@ -301,8 +304,13 @@ class ImageToTensorCalculator : public Node { } } else { if (!cpu_converter_) { +#if !MEDIAPIPE_DISABLE_OPENCV ASSIGN_OR_RETURN(cpu_converter_, CreateOpenCvConverter(cc, GetBorderMode())); +#else + LOG(FATAL) << "Cannot create image to tensor opencv converter since " + "MEDIAPIPE_DISABLE_OPENCV is defined."; +#endif // !MEDIAPIPE_DISABLE_OPENCV } } return absl::OkStatus(); diff --git a/mediapipe/calculators/tensor/image_to_tensor_converter_gl_buffer.cc b/mediapipe/calculators/tensor/image_to_tensor_converter_gl_buffer.cc index 1c27f282..d01916f3 100644 --- a/mediapipe/calculators/tensor/image_to_tensor_converter_gl_buffer.cc +++ b/mediapipe/calculators/tensor/image_to_tensor_converter_gl_buffer.cc @@ -312,7 +312,7 @@ class GlProcessor : public ImageToTensorConverter { return absl::OkStatus(); })); - return std::move(tensor); + return tensor; } ~GlProcessor() override { @@ -338,8 +338,7 @@ CreateImageToGlBufferTensorConverter(CalculatorContext* cc, auto result = absl::make_unique(); MP_RETURN_IF_ERROR(result->Init(cc, input_starts_at_bottom, border_mode)); - // Simply "return std::move(result)" failed to build on macOS with bazel. - return std::unique_ptr(std::move(result)); + return result; } } // namespace mediapipe diff --git a/mediapipe/calculators/tensor/image_to_tensor_converter_gl_texture.cc b/mediapipe/calculators/tensor/image_to_tensor_converter_gl_texture.cc index 26c31eaf..eb968152 100644 --- a/mediapipe/calculators/tensor/image_to_tensor_converter_gl_texture.cc +++ b/mediapipe/calculators/tensor/image_to_tensor_converter_gl_texture.cc @@ -334,9 +334,7 @@ CreateImageToGlTextureTensorConverter(CalculatorContext* cc, BorderMode border_mode) { auto result = absl::make_unique(); MP_RETURN_IF_ERROR(result->Init(cc, input_starts_at_bottom, border_mode)); - - // Simply "return std::move(result)" failed to build on macOS with bazel. - return std::unique_ptr(std::move(result)); + return result; } } // namespace mediapipe diff --git a/mediapipe/calculators/tensor/image_to_tensor_converter_metal.cc b/mediapipe/calculators/tensor/image_to_tensor_converter_metal.cc index 1f86e1ce..9714faa5 100644 --- a/mediapipe/calculators/tensor/image_to_tensor_converter_metal.cc +++ b/mediapipe/calculators/tensor/image_to_tensor_converter_metal.cc @@ -383,7 +383,7 @@ class MetalProcessor : public ImageToTensorConverter { tflite::gpu::HW(output_dims.height, output_dims.width), command_buffer, buffer_view.buffer())); [command_buffer commit]; - return std::move(tensor); + return tensor; } } @@ -399,8 +399,7 @@ absl::StatusOr> CreateMetalConverter( auto result = absl::make_unique(); MP_RETURN_IF_ERROR(result->Init(cc, border_mode)); - // Simply "return std::move(result)" failed to build on macOS with bazel. - return std::unique_ptr(std::move(result)); + return result; } } // namespace mediapipe diff --git a/mediapipe/calculators/tensor/image_to_tensor_converter_opencv.cc b/mediapipe/calculators/tensor/image_to_tensor_converter_opencv.cc index 04a4bbd9..22131a7e 100644 --- a/mediapipe/calculators/tensor/image_to_tensor_converter_opencv.cc +++ b/mediapipe/calculators/tensor/image_to_tensor_converter_opencv.cc @@ -103,7 +103,7 @@ class OpenCvProcessor : public ImageToTensorConverter { GetValueRangeTransformation(kInputImageRangeMin, kInputImageRangeMax, range_min, range_max)); transformed.convertTo(dst, CV_32FC3, transform.scale, transform.offset); - return std::move(tensor); + return tensor; } private: @@ -114,10 +114,7 @@ class OpenCvProcessor : public ImageToTensorConverter { absl::StatusOr> CreateOpenCvConverter( CalculatorContext* cc, BorderMode border_mode) { - // Simply "return absl::make_unique()" failed to build on - // macOS with bazel. - return std::unique_ptr( - absl::make_unique(border_mode)); + return absl::make_unique(border_mode); } } // namespace mediapipe diff --git a/mediapipe/calculators/tensor/testdata/face_detection_test.pbtxt b/mediapipe/calculators/tensor/testdata/face_detection_test.pbtxt index b0e00346..64d970e1 100644 --- a/mediapipe/calculators/tensor/testdata/face_detection_test.pbtxt +++ b/mediapipe/calculators/tensor/testdata/face_detection_test.pbtxt @@ -4,7 +4,7 @@ output_stream: "detections" # Subgraph that detects faces. node { - calculator: "FaceDetectionFrontCpu" + calculator: "FaceDetectionShortRangeCpu" input_stream: "IMAGE:image" output_stream: "DETECTIONS:detections" } diff --git a/mediapipe/calculators/tensorflow/tensorflow_inference_calculator.cc b/mediapipe/calculators/tensorflow/tensorflow_inference_calculator.cc index 625612c1..fc81207a 100644 --- a/mediapipe/calculators/tensorflow/tensorflow_inference_calculator.cc +++ b/mediapipe/calculators/tensorflow/tensorflow_inference_calculator.cc @@ -490,7 +490,7 @@ class TensorFlowInferenceCalculator : public CalculatorBase { << keyed_tensors.first; } } else { - // Pad by replicating the first tens or, then ignore the values. + // Pad by replicating the first tensor, then ignore the values. keyed_tensors.second.resize(options_.batch_size()); std::fill(keyed_tensors.second.begin() + inference_state->batch_timestamps_.size(), diff --git a/mediapipe/calculators/util/BUILD b/mediapipe/calculators/util/BUILD index 62455f01..e759ff99 100644 --- a/mediapipe/calculators/util/BUILD +++ b/mediapipe/calculators/util/BUILD @@ -840,6 +840,20 @@ cc_test( ], ) +cc_library( + name = "world_landmark_projection_calculator", + srcs = ["world_landmark_projection_calculator.cc"], + visibility = ["//visibility:public"], + deps = [ + "//mediapipe/framework:calculator_framework", + "//mediapipe/framework/formats:landmark_cc_proto", + "//mediapipe/framework/formats:rect_cc_proto", + "//mediapipe/framework/port:ret_check", + "//mediapipe/framework/port:status", + ], + alwayslink = 1, +) + mediapipe_proto_library( name = "landmarks_smoothing_calculator_proto", srcs = ["landmarks_smoothing_calculator.proto"], @@ -894,6 +908,31 @@ cc_library( alwayslink = 1, ) +mediapipe_proto_library( + name = "visibility_copy_calculator_proto", + srcs = ["visibility_copy_calculator.proto"], + visibility = ["//visibility:public"], + deps = [ + "//mediapipe/framework:calculator_options_proto", + "//mediapipe/framework:calculator_proto", + ], +) + +cc_library( + name = "visibility_copy_calculator", + srcs = ["visibility_copy_calculator.cc"], + visibility = ["//visibility:public"], + deps = [ + ":visibility_copy_calculator_cc_proto", + "//mediapipe/framework:calculator_framework", + "//mediapipe/framework:timestamp", + "//mediapipe/framework/formats:landmark_cc_proto", + "//mediapipe/framework/port:ret_check", + "@com_google_absl//absl/algorithm:container", + ], + alwayslink = 1, +) + cc_library( name = "landmarks_to_floats_calculator", srcs = ["landmarks_to_floats_calculator.cc"], diff --git a/mediapipe/calculators/util/annotation_overlay_calculator.cc b/mediapipe/calculators/util/annotation_overlay_calculator.cc index 2c0b2539..8af4a5de 100644 --- a/mediapipe/calculators/util/annotation_overlay_calculator.cc +++ b/mediapipe/calculators/util/annotation_overlay_calculator.cc @@ -272,6 +272,15 @@ absl::Status AnnotationOverlayCalculator::Open(CalculatorContext* cc) { } absl::Status AnnotationOverlayCalculator::Process(CalculatorContext* cc) { + if (cc->Inputs().HasTag(kGpuBufferTag) && + cc->Inputs().Tag(kGpuBufferTag).IsEmpty()) { + return absl::OkStatus(); + } + if (cc->Inputs().HasTag(kImageFrameTag) && + cc->Inputs().Tag(kImageFrameTag).IsEmpty()) { + return absl::OkStatus(); + } + // Initialize render target, drawn with OpenCV. std::unique_ptr image_mat; ImageFormat::Format target_format; diff --git a/mediapipe/calculators/util/detections_to_rects_calculator.cc b/mediapipe/calculators/util/detections_to_rects_calculator.cc index a1b13a8d..73a67d32 100644 --- a/mediapipe/calculators/util/detections_to_rects_calculator.cc +++ b/mediapipe/calculators/util/detections_to_rects_calculator.cc @@ -203,6 +203,9 @@ absl::Status DetectionsToRectsCalculator::Process(CalculatorContext* cc) { cc->Inputs().Tag(kDetectionsTag).IsEmpty()) { return absl::OkStatus(); } + if (rotate_ && !HasTagValue(cc, kImageSizeTag)) { + return absl::OkStatus(); + } std::vector detections; if (cc->Inputs().HasTag(kDetectionTag)) { diff --git a/mediapipe/calculators/util/rect_transformation_calculator.cc b/mediapipe/calculators/util/rect_transformation_calculator.cc index 7c71dd5a..e0a759bd 100644 --- a/mediapipe/calculators/util/rect_transformation_calculator.cc +++ b/mediapipe/calculators/util/rect_transformation_calculator.cc @@ -130,8 +130,8 @@ absl::Status RectTransformationCalculator::Process(CalculatorContext* cc) { } cc->Outputs().Index(0).Add(output_rects.release(), cc->InputTimestamp()); } - if (cc->Inputs().HasTag(kNormRectTag) && - !cc->Inputs().Tag(kNormRectTag).IsEmpty()) { + if (HasTagValue(cc->Inputs(), kNormRectTag) && + HasTagValue(cc->Inputs(), kImageSizeTag)) { auto rect = cc->Inputs().Tag(kNormRectTag).Get(); const auto& image_size = cc->Inputs().Tag(kImageSizeTag).Get>(); @@ -139,8 +139,8 @@ absl::Status RectTransformationCalculator::Process(CalculatorContext* cc) { cc->Outputs().Index(0).AddPacket( MakePacket(rect).At(cc->InputTimestamp())); } - if (cc->Inputs().HasTag(kNormRectsTag) && - !cc->Inputs().Tag(kNormRectsTag).IsEmpty()) { + if (HasTagValue(cc->Inputs(), kNormRectsTag) && + HasTagValue(cc->Inputs(), kImageSizeTag)) { auto rects = cc->Inputs().Tag(kNormRectsTag).Get>(); const auto& image_size = diff --git a/mediapipe/calculators/video/motion_analysis_calculator.cc b/mediapipe/calculators/video/motion_analysis_calculator.cc index 59673108..4e8ddac4 100644 --- a/mediapipe/calculators/video/motion_analysis_calculator.cc +++ b/mediapipe/calculators/video/motion_analysis_calculator.cc @@ -549,7 +549,7 @@ absl::Status MotionAnalysisCalculator::Process(CalculatorContext* cc) { timestamp_buffer_.push_back(timestamp); ++frame_idx_; - VLOG_EVERY_N(0, 100) << "Analyzed frame " << frame_idx_; + VLOG_EVERY_N(1, 100) << "Analyzed frame " << frame_idx_; // Buffer input frames only if visualization is requested. if (visualize_output_ || video_output_) { diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectioncpu/BUILD b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectioncpu/BUILD index 279d29b7..cf71048b 100644 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectioncpu/BUILD +++ b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectioncpu/BUILD @@ -37,7 +37,7 @@ android_binary( srcs = glob(["*.java"]), assets = [ "//mediapipe/graphs/face_detection:face_detection_mobile_cpu.binarypb", - "//mediapipe/modules/face_detection:face_detection_front.tflite", + "//mediapipe/modules/face_detection:face_detection_short_range.tflite", ], assets_dir = "", manifest = "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:AndroidManifest.xml", diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectionfullrangegpu/BUILD b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectionfullrangegpu/BUILD new file mode 100644 index 00000000..03891439 --- /dev/null +++ b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectionfullrangegpu/BUILD @@ -0,0 +1,60 @@ +# Copyright 2019 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +licenses(["notice"]) + +package(default_visibility = ["//visibility:private"]) + +cc_binary( + name = "libmediapipe_jni.so", + linkshared = 1, + linkstatic = 1, + deps = [ + "//mediapipe/graphs/face_detection:face_detection_full_range_mobile_gpu_deps", + "//mediapipe/java/com/google/mediapipe/framework/jni:mediapipe_framework_jni", + ], +) + +cc_library( + name = "mediapipe_jni_lib", + srcs = [":libmediapipe_jni.so"], + alwayslink = 1, +) + +android_binary( + name = "facedetectionfullrangegpu", + srcs = glob(["*.java"]), + assets = [ + "//mediapipe/graphs/face_detection:face_detection_full_range_mobile_gpu.binarypb", + "//mediapipe/modules/face_detection:face_detection_full_range_sparse.tflite", + ], + assets_dir = "", + manifest = "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:AndroidManifest.xml", + manifest_values = { + "applicationId": "com.google.mediapipe.apps.facedetectionfullrangegpu", + "appName": "Face Detection Full-range (GPU)", + "mainActivity": "com.google.mediapipe.apps.basic.MainActivity", + "cameraFacingFront": "False", + "binaryGraphName": "face_detection_full_range_mobile_gpu.binarypb", + "inputVideoStreamName": "input_video", + "outputVideoStreamName": "output_video", + "flipFramesVertically": "True", + "converterNumBuffers": "2", + }, + multidex = "native", + deps = [ + ":mediapipe_jni_lib", + "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:basic_lib", + ], +) diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectiongpu/BUILD b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectiongpu/BUILD index 11351fc5..9c60d5a5 100644 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectiongpu/BUILD +++ b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectiongpu/BUILD @@ -37,7 +37,7 @@ android_binary( srcs = glob(["*.java"]), assets = [ "//mediapipe/graphs/face_detection:face_detection_mobile_gpu.binarypb", - "//mediapipe/modules/face_detection:face_detection_front.tflite", + "//mediapipe/modules/face_detection:face_detection_short_range.tflite", ], assets_dir = "", manifest = "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:AndroidManifest.xml", diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/faceeffect/BUILD b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/faceeffect/BUILD index 8bf6c0a5..becef524 100644 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/faceeffect/BUILD +++ b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/faceeffect/BUILD @@ -42,7 +42,7 @@ android_binary( "//mediapipe/graphs/face_effect/data:glasses.binarypb", "//mediapipe/graphs/face_effect/data:glasses.pngblob", "//mediapipe/graphs/face_effect:face_effect_gpu.binarypb", - "//mediapipe/modules/face_detection:face_detection_front.tflite", + "//mediapipe/modules/face_detection:face_detection_short_range.tflite", "//mediapipe/modules/face_geometry/data:geometry_pipeline_metadata_detection.binarypb", "//mediapipe/modules/face_geometry/data:geometry_pipeline_metadata_landmarks.binarypb", "//mediapipe/modules/face_landmark:face_landmark.tflite", diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facemeshgpu/BUILD b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facemeshgpu/BUILD index 26406e77..378132c1 100644 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facemeshgpu/BUILD +++ b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facemeshgpu/BUILD @@ -38,7 +38,7 @@ android_binary( assets = [ "//mediapipe/graphs/face_mesh:face_mesh_mobile_gpu.binarypb", "//mediapipe/modules/face_landmark:face_landmark.tflite", - "//mediapipe/modules/face_detection:face_detection_front.tflite", + "//mediapipe/modules/face_detection:face_detection_short_range.tflite", ], assets_dir = "", manifest = "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:AndroidManifest.xml", diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/holistictrackinggpu/BUILD b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/holistictrackinggpu/BUILD index d9c66008..a2e4839f 100644 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/holistictrackinggpu/BUILD +++ b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/holistictrackinggpu/BUILD @@ -37,7 +37,7 @@ android_binary( srcs = glob(["*.java"]), assets = [ "//mediapipe/graphs/holistic_tracking:holistic_tracking_gpu.binarypb", - "//mediapipe/modules/face_detection:face_detection_front.tflite", + "//mediapipe/modules/face_detection:face_detection_short_range.tflite", "//mediapipe/modules/face_landmark:face_landmark.tflite", "//mediapipe/modules/hand_landmark:hand_landmark.tflite", "//mediapipe/modules/hand_landmark:handedness.txt", diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/iristrackinggpu/BUILD b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/iristrackinggpu/BUILD index f629951d..482932b1 100644 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/iristrackinggpu/BUILD +++ b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/iristrackinggpu/BUILD @@ -39,7 +39,7 @@ android_binary( "//mediapipe/graphs/iris_tracking:iris_tracking_gpu.binarypb", "//mediapipe/modules/face_landmark:face_landmark.tflite", "//mediapipe/modules/iris_landmark:iris_landmark.tflite", - "//mediapipe/modules/face_detection:face_detection_front.tflite", + "//mediapipe/modules/face_detection:face_detection_short_range.tflite", ], assets_dir = "", manifest = "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:AndroidManifest.xml", diff --git a/mediapipe/examples/desktop/autoflip/subgraph/face_detection_subgraph.pbtxt b/mediapipe/examples/desktop/autoflip/subgraph/face_detection_subgraph.pbtxt index 4024f355..b736a157 100644 --- a/mediapipe/examples/desktop/autoflip/subgraph/face_detection_subgraph.pbtxt +++ b/mediapipe/examples/desktop/autoflip/subgraph/face_detection_subgraph.pbtxt @@ -38,7 +38,7 @@ node { output_stream: "TENSORS:detection_tensors" options: { [mediapipe.TfLiteInferenceCalculatorOptions.ext] { - model_path: "mediapipe/modules/face_detection/face_detection_back.tflite" + model_path: "mediapipe/modules/face_detection/face_detection_full_range_sparse.tflite" } } } diff --git a/mediapipe/examples/desktop/face_detection/BUILD b/mediapipe/examples/desktop/face_detection/BUILD index 5743ae78..8cd75b44 100644 --- a/mediapipe/examples/desktop/face_detection/BUILD +++ b/mediapipe/examples/desktop/face_detection/BUILD @@ -16,6 +16,14 @@ licenses(["notice"]) package(default_visibility = ["//mediapipe/examples:__subpackages__"]) +cc_binary( + name = "face_detection_full_range_cpu", + deps = [ + "//mediapipe/examples/desktop:demo_run_graph_main", + "//mediapipe/graphs/face_detection:face_detection_full_range_desktop_live_deps", + ], +) + cc_binary( name = "face_detection_cpu", deps = [ diff --git a/mediapipe/examples/ios/facedetectioncpu/BUILD b/mediapipe/examples/ios/facedetectioncpu/BUILD index 43bff9b1..0acd41df 100644 --- a/mediapipe/examples/ios/facedetectioncpu/BUILD +++ b/mediapipe/examples/ios/facedetectioncpu/BUILD @@ -55,7 +55,7 @@ objc_library( name = "FaceDetectionCpuAppLibrary", data = [ "//mediapipe/graphs/face_detection:face_detection_mobile_cpu.binarypb", - "//mediapipe/modules/face_detection:face_detection_front.tflite", + "//mediapipe/modules/face_detection:face_detection_short_range.tflite", ], deps = [ "//mediapipe/examples/ios/common:CommonMediaPipeAppLibrary", diff --git a/mediapipe/examples/ios/facedetectiongpu/BUILD b/mediapipe/examples/ios/facedetectiongpu/BUILD index 51856a7f..4ca3c267 100644 --- a/mediapipe/examples/ios/facedetectiongpu/BUILD +++ b/mediapipe/examples/ios/facedetectiongpu/BUILD @@ -55,7 +55,7 @@ objc_library( name = "FaceDetectionGpuAppLibrary", data = [ "//mediapipe/graphs/face_detection:face_detection_mobile_gpu.binarypb", - "//mediapipe/modules/face_detection:face_detection_front.tflite", + "//mediapipe/modules/face_detection:face_detection_short_range.tflite", ], deps = [ "//mediapipe/examples/ios/common:CommonMediaPipeAppLibrary", diff --git a/mediapipe/examples/ios/faceeffect/BUILD b/mediapipe/examples/ios/faceeffect/BUILD index 9e074ef2..5dafa93e 100644 --- a/mediapipe/examples/ios/faceeffect/BUILD +++ b/mediapipe/examples/ios/faceeffect/BUILD @@ -66,7 +66,7 @@ objc_library( "//mediapipe/graphs/face_effect/data:facepaint.pngblob", "//mediapipe/graphs/face_effect/data:glasses.binarypb", "//mediapipe/graphs/face_effect/data:glasses.pngblob", - "//mediapipe/modules/face_detection:face_detection_front.tflite", + "//mediapipe/modules/face_detection:face_detection_short_range.tflite", "//mediapipe/modules/face_geometry/data:geometry_pipeline_metadata.binarypb", "//mediapipe/modules/face_geometry/data:geometry_pipeline_metadata_detection.binarypb", "//mediapipe/modules/face_geometry/data:geometry_pipeline_metadata_landmarks.binarypb", @@ -109,7 +109,7 @@ objc_library( "//mediapipe/graphs/face_effect/data:facepaint.pngblob", "//mediapipe/graphs/face_effect/data:glasses.binarypb", "//mediapipe/graphs/face_effect/data:glasses.pngblob", - "//mediapipe/modules/face_detection:face_detection_front.tflite", + "//mediapipe/modules/face_detection:face_detection_short_range.tflite", "//mediapipe/modules/face_geometry/data:geometry_pipeline_metadata.binarypb", "//mediapipe/modules/face_landmark:face_landmark.tflite", ], diff --git a/mediapipe/examples/ios/facemeshgpu/BUILD b/mediapipe/examples/ios/facemeshgpu/BUILD index 942a1965..a8d5ef66 100644 --- a/mediapipe/examples/ios/facemeshgpu/BUILD +++ b/mediapipe/examples/ios/facemeshgpu/BUILD @@ -62,7 +62,7 @@ objc_library( copts = ["-std=c++17"], data = [ "//mediapipe/graphs/face_mesh:face_mesh_mobile_gpu.binarypb", - "//mediapipe/modules/face_detection:face_detection_front.tflite", + "//mediapipe/modules/face_detection:face_detection_short_range.tflite", "//mediapipe/modules/face_landmark:face_landmark.tflite", ], deps = [ diff --git a/mediapipe/examples/ios/holistictrackinggpu/BUILD b/mediapipe/examples/ios/holistictrackinggpu/BUILD index b080564c..b2a58913 100644 --- a/mediapipe/examples/ios/holistictrackinggpu/BUILD +++ b/mediapipe/examples/ios/holistictrackinggpu/BUILD @@ -55,7 +55,7 @@ objc_library( name = "HolisticTrackingGpuAppLibrary", data = [ "//mediapipe/graphs/holistic_tracking:holistic_tracking_gpu.binarypb", - "//mediapipe/modules/face_detection:face_detection_front.tflite", + "//mediapipe/modules/face_detection:face_detection_short_range.tflite", "//mediapipe/modules/face_landmark:face_landmark.tflite", "//mediapipe/modules/hand_landmark:hand_landmark.tflite", "//mediapipe/modules/hand_landmark:handedness.txt", diff --git a/mediapipe/examples/ios/iristrackinggpu/BUILD b/mediapipe/examples/ios/iristrackinggpu/BUILD index b58ecc10..84222760 100644 --- a/mediapipe/examples/ios/iristrackinggpu/BUILD +++ b/mediapipe/examples/ios/iristrackinggpu/BUILD @@ -62,7 +62,7 @@ objc_library( copts = ["-std=c++17"], data = [ "//mediapipe/graphs/iris_tracking:iris_tracking_gpu.binarypb", - "//mediapipe/modules/face_detection:face_detection_front.tflite", + "//mediapipe/modules/face_detection:face_detection_short_range.tflite", "//mediapipe/modules/face_landmark:face_landmark.tflite", "//mediapipe/modules/iris_landmark:iris_landmark.tflite", ], diff --git a/mediapipe/framework/BUILD b/mediapipe/framework/BUILD index 109625bb..c25e9d8f 100644 --- a/mediapipe/framework/BUILD +++ b/mediapipe/framework/BUILD @@ -953,6 +953,9 @@ cc_library( }) + select({ "//conditions:default": [], "//mediapipe/gpu:disable_gpu": ["MEDIAPIPE_DISABLE_GPU=1"], + }) + select({ + "//conditions:default": [], + "//mediapipe/framework/port:disable_opencv": ["MEDIAPIPE_DISABLE_OPENCV=1"], }) + select({ "//conditions:default": [], "//mediapipe/framework:disable_rtti_and_exceptions": [ diff --git a/mediapipe/framework/api2/builder.h b/mediapipe/framework/api2/builder.h index ae32c628..f1cbfded 100644 --- a/mediapipe/framework/api2/builder.h +++ b/mediapipe/framework/api2/builder.h @@ -17,6 +17,14 @@ namespace mediapipe { namespace api2 { namespace builder { +// Workaround for static_assert(false). Example: +// dependent_false::value returns false. +// For more information, see: +// https://en.cppreference.com/w/cpp/language/if#Constexpr_If +// TODO: migrate to a common utility when available. +template +struct dependent_false : std::false_type {}; + template T& GetWithAutoGrow(std::vector>* vecp, int index) { auto& vec = *vecp; @@ -209,6 +217,21 @@ class NodeBase { return SideDestination(&in_sides_[tag]); } + template + auto operator[](const PortCommon& port) { + if constexpr (std::is_same_v) { + return Source(&out_streams_[port.Tag()]); + } else if constexpr (std::is_same_v) { + return Destination(&in_streams_[port.Tag()]); + } else if constexpr (std::is_same_v) { + return SideSource(&out_sides_[port.Tag()]); + } else if constexpr (std::is_same_v) { + return SideDestination(&in_sides_[port.Tag()]); + } else { + static_assert(dependent_false::value, "Type not supported."); + } + } + // Convenience methods for accessing purely index-based ports. Source Out(int index) { return Out("")[index]; } @@ -429,6 +452,24 @@ class Graph { return Dst(&graph_boundary_.in_sides_[graph_output.Tag()]); } + template + auto operator[](const PortCommon& port) { + if constexpr (std::is_same_v) { + return Destination( + &graph_boundary_.in_streams_[port.Tag()]); + } else if constexpr (std::is_same_v) { + return Source(&graph_boundary_.out_streams_[port.Tag()]); + } else if constexpr (std::is_same_v) { + return SideDestination( + &graph_boundary_.in_sides_[port.Tag()]); + } else if constexpr (std::is_same_v) { + return SideSource( + &graph_boundary_.out_sides_[port.Tag()]); + } else { + static_assert(dependent_false::value, "Type not supported."); + } + } + // Returns the graph config. This can be used to instantiate and run the // graph. CalculatorGraphConfig GetConfig() { diff --git a/mediapipe/framework/api2/builder_test.cc b/mediapipe/framework/api2/builder_test.cc index e0e29f01..32d16a9e 100644 --- a/mediapipe/framework/api2/builder_test.cc +++ b/mediapipe/framework/api2/builder_test.cc @@ -138,6 +138,35 @@ TEST(BuilderTest, TypedMultiple) { EXPECT_THAT(graph.GetConfig(), EqualsProto(expected)); } +TEST(BuilderTest, TypedByPorts) { + builder::Graph graph; + auto& foo = graph.AddNode(); + auto& adder = graph.AddNode(); + + graph[FooBar1::kIn].SetName("base") >> foo[Foo::kBase]; + foo[Foo::kOut] >> adder[FloatAdder::kIn][0]; + foo[Foo::kOut] >> adder[FloatAdder::kIn][1]; + adder[FloatAdder::kOut].SetName("out") >> graph[FooBar1::kOut]; + + CalculatorGraphConfig expected = + mediapipe::ParseTextProtoOrDie(R"pb( + input_stream: "IN:base" + output_stream: "OUT:out" + node { + calculator: "Foo" + input_stream: "BASE:base" + output_stream: "OUT:__stream_0" + } + node { + calculator: "FloatAdder" + input_stream: "IN:0:__stream_0" + input_stream: "IN:1:__stream_0" + output_stream: "OUT:out" + } + )pb"); + EXPECT_THAT(graph.GetConfig(), EqualsProto(expected)); +} + TEST(BuilderTest, PacketGenerator) { builder::Graph graph; auto& generator = graph.AddPacketGenerator("FloatGenerator"); diff --git a/mediapipe/framework/formats/BUILD b/mediapipe/framework/formats/BUILD index 902524e1..e0ec40e9 100644 --- a/mediapipe/framework/formats/BUILD +++ b/mediapipe/framework/formats/BUILD @@ -186,6 +186,7 @@ cc_library( "//conditions:default": [ "//mediapipe/framework/port:opencv_imgproc", ], + "//mediapipe/framework/port:disable_opencv": [], }) + select({ "//conditions:default": [ ], diff --git a/mediapipe/framework/formats/image.cc b/mediapipe/framework/formats/image.cc index 0591c3c6..9d50f1f2 100644 --- a/mediapipe/framework/formats/image.cc +++ b/mediapipe/framework/formats/image.cc @@ -76,10 +76,7 @@ bool Image::ConvertToGpu() const { gpu_buffer_ = mediapipe::GpuBuffer(std::move(buffer)); #else // GlCalculatorHelperImpl::MakeGlTextureBuffer (CreateSourceTexture) - auto buffer = mediapipe::GlTextureBuffer::Create( - image_frame_->Width(), image_frame_->Height(), - mediapipe::GpuBufferFormatForImageFormat(image_frame_->Format()), - image_frame_->PixelData()); + auto buffer = mediapipe::GlTextureBuffer::Create(*image_frame_); glBindTexture(GL_TEXTURE_2D, buffer->name()); // See GlCalculatorHelperImpl::SetStandardTextureParams glTexParameteri(buffer->target(), GL_TEXTURE_MIN_FILTER, GL_LINEAR); diff --git a/mediapipe/framework/formats/location.h b/mediapipe/framework/formats/location.h index a9c2aafa..e75ff90b 100644 --- a/mediapipe/framework/formats/location.h +++ b/mediapipe/framework/formats/location.h @@ -32,7 +32,12 @@ // clang-format off #if !defined(LOCATION_OPENCV) -# define LOCATION_OPENCV 1 +# if !MEDIAPIPE_DISABLE_OPENCV && \ + (!defined(MEDIAPIPE_MOBILE) || defined(MEDIAPIPE_ANDROID_OPENCV)) +# define LOCATION_OPENCV 1 +# else +# define LOCATION_OPENCV 0 +# endif #endif #if LOCATION_OPENCV diff --git a/mediapipe/framework/port/BUILD b/mediapipe/framework/port/BUILD index a6827d2e..86c9ff30 100644 --- a/mediapipe/framework/port/BUILD +++ b/mediapipe/framework/port/BUILD @@ -158,12 +158,12 @@ cc_library( hdrs = [ "gmock.h", "gtest.h", + "gtest-spi.h", "status_matchers.h", ], visibility = ["//visibility:public"], deps = [ ":status_matchers", - "//mediapipe/framework:port", "@com_google_googletest//:gtest", ], ) @@ -174,12 +174,12 @@ cc_library( hdrs = [ "gmock.h", "gtest.h", + "gtest-spi.h", "status_matchers.h", ], visibility = ["//visibility:public"], deps = [ ":status_matchers", - "//mediapipe/framework:port", "//mediapipe/framework/deps:status_matchers", "@com_google_googletest//:gtest_main", ], @@ -217,6 +217,16 @@ cc_library( deps = ["//mediapipe/framework/deps:numbers"], ) +# Disabling opencv when defining MEDIAPIPE_DISABLE_OPENCV to 1 in the bazel command. +# Note that this only applies to a select few calculators/framework components currently. +config_setting( + name = "disable_opencv", + define_values = { + "MEDIAPIPE_DISABLE_OPENCV": "1", + }, + visibility = ["//visibility:public"], +) + cc_library( name = "opencv_core", hdrs = ["opencv_core_inc.h"], diff --git a/mediapipe/framework/port/gtest-spi.h b/mediapipe/framework/port/gtest-spi.h new file mode 100644 index 00000000..14031fb4 --- /dev/null +++ b/mediapipe/framework/port/gtest-spi.h @@ -0,0 +1,20 @@ +// Copyright 2021 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MEDIAPIPE_PORT_GTEST_SPI_H_ +#define MEDIAPIPE_PORT_GTEST_SPI_H_ + +#include "gtest/gtest-spi.h" + +#endif // MEDIAPIPE_PORT_GTEST_SPI_H_ diff --git a/mediapipe/gpu/BUILD b/mediapipe/gpu/BUILD index fbcdb4ab..da53b98a 100644 --- a/mediapipe/gpu/BUILD +++ b/mediapipe/gpu/BUILD @@ -585,6 +585,7 @@ cc_library( "//mediapipe:apple": [ ":gl_calculator_helper_ios", "//mediapipe/objc:util", + "//mediapipe/objc:CFHolder", ], }), ) @@ -714,11 +715,12 @@ cc_library( deps = [ ":gl_calculator_helper", "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:timestamp", "//mediapipe/framework/formats:image_frame", - "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", - ], + ] + select({ + "//conditions:default": [], + "//mediapipe:apple": ["//mediapipe/objc:util"], + }), alwayslink = 1, ) diff --git a/mediapipe/gpu/gl_calculator_helper_impl_common.cc b/mediapipe/gpu/gl_calculator_helper_impl_common.cc index 2e4ab10b..fb2685bc 100644 --- a/mediapipe/gpu/gl_calculator_helper_impl_common.cc +++ b/mediapipe/gpu/gl_calculator_helper_impl_common.cc @@ -12,6 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include + +#include "mediapipe/framework/formats/image_frame.h" #include "mediapipe/gpu/gl_calculator_helper_impl.h" #include "mediapipe/gpu/gpu_buffer_format.h" #include "mediapipe/gpu/gpu_shared_data_internal.h" @@ -176,10 +179,8 @@ GlTexture GlCalculatorHelperImpl::MapGlTextureBuffer( GlTextureBufferSharedPtr GlCalculatorHelperImpl::MakeGlTextureBuffer( const ImageFrame& image_frame) { CHECK(gl_context_->IsCurrent()); - auto buffer = GlTextureBuffer::Create( - image_frame.Width(), image_frame.Height(), - GpuBufferFormatForImageFormat(image_frame.Format()), - image_frame.PixelData()); + + auto buffer = GlTextureBuffer::Create(image_frame); if (buffer->format_ != GpuBufferFormat::kUnknown) { glBindTexture(GL_TEXTURE_2D, buffer->name_); diff --git a/mediapipe/gpu/gl_texture_buffer.cc b/mediapipe/gpu/gl_texture_buffer.cc index 079c821f..d1c13e04 100644 --- a/mediapipe/gpu/gl_texture_buffer.cc +++ b/mediapipe/gpu/gl_texture_buffer.cc @@ -32,15 +32,56 @@ std::unique_ptr GlTextureBuffer::Wrap( std::unique_ptr GlTextureBuffer::Create(int width, int height, GpuBufferFormat format, - const void* data) { + const void* data, + int alignment) { auto buf = absl::make_unique(GL_TEXTURE_2D, 0, width, height, format, nullptr); - if (!buf->CreateInternal(data)) { + if (!buf->CreateInternal(data, alignment)) { return nullptr; } return buf; } +static inline int AlignedToPowerOf2(int value, int alignment) { + // alignment must be a power of 2 + return ((value - 1) | (alignment - 1)) + 1; +} + +std::unique_ptr GlTextureBuffer::Create( + const ImageFrame& image_frame) { + int base_ws = image_frame.Width() * image_frame.NumberOfChannels() * + image_frame.ByteDepth(); + int actual_ws = image_frame.WidthStep(); + int alignment = 0; + std::unique_ptr temp; + const uint8* data = image_frame.PixelData(); + + // Let's see if the pixel data is tightly aligned to one of the alignments + // supported by OpenGL, preferring 4 if possible since it's the default. + if (actual_ws == AlignedToPowerOf2(base_ws, 4)) + alignment = 4; + else if (actual_ws == AlignedToPowerOf2(base_ws, 1)) + alignment = 1; + else if (actual_ws == AlignedToPowerOf2(base_ws, 2)) + alignment = 2; + else if (actual_ws == AlignedToPowerOf2(base_ws, 8)) + alignment = 8; + + // If no GL-compatible alignment was found, we copy the data to a temporary + // buffer, aligned to 4. We do this using another ImageFrame purely for + // convenience. + if (!alignment) { + temp = std::make_unique(); + temp->CopyFrom(image_frame, 4); + data = temp->PixelData(); + alignment = 4; + } + + return Create(image_frame.Width(), image_frame.Height(), + GpuBufferFormatForImageFormat(image_frame.Format()), data, + alignment); +} + GlTextureBuffer::GlTextureBuffer(GLenum target, GLuint name, int width, int height, GpuBufferFormat format, DeletionCallback deletion_callback, @@ -53,7 +94,7 @@ GlTextureBuffer::GlTextureBuffer(GLenum target, GLuint name, int width, deletion_callback_(deletion_callback), producer_context_(producer_context) {} -bool GlTextureBuffer::CreateInternal(const void* data) { +bool GlTextureBuffer::CreateInternal(const void* data, int alignment) { auto context = GlContext::GetCurrent(); if (!context) return false; @@ -66,8 +107,11 @@ bool GlTextureBuffer::CreateInternal(const void* data) { GlTextureInfo info = GlTextureInfoForGpuBufferFormat(format_, 0, context->GetGlVersion()); + if (alignment != 4 && data) glPixelStorei(GL_UNPACK_ALIGNMENT, alignment); + // See b/70294573 for details about this. if (info.gl_internal_format == GL_RGBA16F && + context->GetGlVersion() != GlVersion::kGLES2 && SymbolAvailable(&glTexStorage2D)) { CHECK(data == nullptr) << "unimplemented"; glTexStorage2D(target_, 1, info.gl_internal_format, width_, height_); @@ -76,6 +120,8 @@ bool GlTextureBuffer::CreateInternal(const void* data) { height_, 0 /* border */, info.gl_format, info.gl_type, data); } + if (alignment != 4 && data) glPixelStorei(GL_UNPACK_ALIGNMENT, 4); + glBindTexture(target_, 0); // Use the deletion callback to delete the texture on the context diff --git a/mediapipe/gpu/gl_texture_buffer.h b/mediapipe/gpu/gl_texture_buffer.h index e6c44b6f..206148a4 100644 --- a/mediapipe/gpu/gl_texture_buffer.h +++ b/mediapipe/gpu/gl_texture_buffer.h @@ -21,6 +21,7 @@ #include #include "absl/memory/memory.h" +#include "mediapipe/framework/formats/image_frame.h" #include "mediapipe/gpu/gl_base.h" #include "mediapipe/gpu/gl_context.h" #include "mediapipe/gpu/gpu_buffer_format.h" @@ -60,7 +61,11 @@ class GlTextureBuffer { // provided later via glTexSubImage2D. static std::unique_ptr Create(int width, int height, GpuBufferFormat format, - const void* data = nullptr); + const void* data = nullptr, + int alignment = 4); + + // Create a texture with a copy of the data in image_frame. + static std::unique_ptr Create(const ImageFrame& image_frame); // Wraps an existing texture, but does not take ownership of it. // deletion_callback is invoked when the GlTextureBuffer is released, so @@ -127,7 +132,7 @@ class GlTextureBuffer { // If data is provided, it is uploaded to the texture; otherwise, it can be // provided later via glTexSubImage2D. // Returns true on success. - bool CreateInternal(const void* data = nullptr); + bool CreateInternal(const void* data, int alignment = 4); friend class GlCalculatorHelperImpl; diff --git a/mediapipe/gpu/shader_util.cc b/mediapipe/gpu/shader_util.cc index 8f5edee3..dab781f0 100644 --- a/mediapipe/gpu/shader_util.cc +++ b/mediapipe/gpu/shader_util.cc @@ -51,8 +51,6 @@ namespace mediapipe { constexpr int kMaxShaderInfoLength = 1024; GLint GlhCompileShader(GLenum target, const GLchar* source, GLuint* shader) { - GLint status; - *shader = glCreateShader(target); if (*shader == 0) { return GL_FALSE; @@ -62,6 +60,11 @@ GLint GlhCompileShader(GLenum target, const GLchar* source, GLuint* shader) { GL_DEBUG_LOG(Shader, *shader, "compile"); +#if UNSAFE_EMSCRIPTEN_SKIP_GL_ERROR_HANDLING + return GL_TRUE; +#else + GLint status; + glGetShaderiv(*shader, GL_COMPILE_STATUS, &status); LOG_IF(ERROR, status == GL_FALSE) << "Failed to compile shader:\n" << source; @@ -72,19 +75,24 @@ GLint GlhCompileShader(GLenum target, const GLchar* source, GLuint* shader) { LOG(ERROR) << "Error message: " << std::string(cmessage, length); } return status; +#endif // UNSAFE_EMSCRIPTEN_SKIP_GL_ERROR_HANDLING } GLint GlhLinkProgram(GLuint program) { - GLint status; - glLinkProgram(program); +#if UNSAFE_EMSCRIPTEN_SKIP_GL_ERROR_HANDLING + return GL_TRUE; +#else + GLint status; + GL_DEBUG_LOG(Program, program, "link"); glGetProgramiv(program, GL_LINK_STATUS, &status); LOG_IF(ERROR, status == GL_FALSE) << "Failed to link program " << program; return status; +#endif // UNSAFE_EMSCRIPTEN_SKIP_GL_ERROR_HANDLING } GLint GlhValidateProgram(GLuint program) { diff --git a/mediapipe/graphs/face_detection/BUILD b/mediapipe/graphs/face_detection/BUILD index 40dbc63e..9e7cf250 100644 --- a/mediapipe/graphs/face_detection/BUILD +++ b/mediapipe/graphs/face_detection/BUILD @@ -11,6 +11,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +load( + "//mediapipe/framework/tool:mediapipe_graph.bzl", + "mediapipe_binary_graph", +) licenses(["notice"]) @@ -24,8 +28,8 @@ cc_library( "//mediapipe/calculators/util:detections_to_render_data_calculator", "//mediapipe/gpu:gpu_buffer_to_image_frame_calculator", "//mediapipe/gpu:image_frame_to_gpu_buffer_calculator", - "//mediapipe/modules/face_detection:face_detection_front_cpu", - "//mediapipe/modules/face_detection:face_detection_front_gpu", + "//mediapipe/modules/face_detection:face_detection_short_range_cpu", + "//mediapipe/modules/face_detection:face_detection_short_range_gpu", ], ) @@ -35,7 +39,7 @@ cc_library( "//mediapipe/calculators/core:flow_limiter_calculator", "//mediapipe/calculators/util:annotation_overlay_calculator", "//mediapipe/calculators/util:detections_to_render_data_calculator", - "//mediapipe/modules/face_detection:face_detection_front_cpu", + "//mediapipe/modules/face_detection:face_detection_short_range_cpu", ], ) @@ -45,15 +49,10 @@ cc_library( "//mediapipe/calculators/core:flow_limiter_calculator", "//mediapipe/calculators/util:annotation_overlay_calculator", "//mediapipe/calculators/util:detections_to_render_data_calculator", - "//mediapipe/modules/face_detection:face_detection_front_gpu", + "//mediapipe/modules/face_detection:face_detection_short_range_gpu", ], ) -load( - "//mediapipe/framework/tool:mediapipe_graph.bzl", - "mediapipe_binary_graph", -) - mediapipe_binary_graph( name = "face_detection_mobile_cpu_binary_graph", graph = "face_detection_mobile_cpu.pbtxt", @@ -67,3 +66,30 @@ mediapipe_binary_graph( output_name = "face_detection_mobile_gpu.binarypb", deps = [":mobile_calculators"], ) + +cc_library( + name = "face_detection_full_range_mobile_gpu_deps", + deps = [ + "//mediapipe/calculators/core:flow_limiter_calculator", + "//mediapipe/calculators/util:annotation_overlay_calculator", + "//mediapipe/calculators/util:detections_to_render_data_calculator", + "//mediapipe/modules/face_detection:face_detection_full_range_gpu", + ], +) + +mediapipe_binary_graph( + name = "face_detection_full_range_mobile_gpu_binary_graph", + graph = "face_detection_full_range_mobile_gpu.pbtxt", + output_name = "face_detection_full_range_mobile_gpu.binarypb", + deps = [":face_detection_full_range_mobile_gpu_deps"], +) + +cc_library( + name = "face_detection_full_range_desktop_live_deps", + deps = [ + "//mediapipe/calculators/core:flow_limiter_calculator", + "//mediapipe/calculators/util:annotation_overlay_calculator", + "//mediapipe/calculators/util:detections_to_render_data_calculator", + "//mediapipe/modules/face_detection:face_detection_full_range_cpu", + ], +) diff --git a/mediapipe/graphs/face_detection/face_detection_back_desktop_live.pbtxt b/mediapipe/graphs/face_detection/face_detection_back_desktop_live.pbtxt deleted file mode 100644 index 23a2db27..00000000 --- a/mediapipe/graphs/face_detection/face_detection_back_desktop_live.pbtxt +++ /dev/null @@ -1,169 +0,0 @@ -# MediaPipe graph that performs face detection with TensorFlow Lite on CPU. -# Used in the examples in -# mediapipe/examples/desktop/face_detection:face_detection_cpu. - -# Images on GPU coming into and out of the graph. -input_stream: "input_video" -output_stream: "output_video" - -# Throttles the images flowing downstream for flow control. It passes through -# the very first incoming image unaltered, and waits for -# TfLiteTensorsToDetectionsCalculator downstream in the graph to finish -# generating the corresponding detections before it passes through another -# image. All images that come in while waiting are dropped, limiting the number -# of in-flight images between this calculator and -# TfLiteTensorsToDetectionsCalculator to 1. This prevents the nodes in between -# from queuing up incoming images and data excessively, which leads to increased -# latency and memory usage, unwanted in real-time mobile applications. It also -# eliminates unnecessarily computation, e.g., a transformed image produced by -# ImageTransformationCalculator may get dropped downstream if the subsequent -# TfLiteConverterCalculator or TfLiteInferenceCalculator is still busy -# processing previous inputs. -node { - calculator: "FlowLimiterCalculator" - input_stream: "input_video" - input_stream: "FINISHED:detections" - input_stream_info: { - tag_index: "FINISHED" - back_edge: true - } - output_stream: "throttled_input_video" -} - -# Transforms the input image on CPU to a 128x128 image. To scale the input -# image, the scale_mode option is set to FIT to preserve the aspect ratio, -# resulting in potential letterboxing in the transformed image. -node: { - calculator: "ImageTransformationCalculator" - input_stream: "IMAGE:throttled_input_video" - output_stream: "IMAGE:transformed_input_video_cpu" - output_stream: "LETTERBOX_PADDING:letterbox_padding" - node_options: { - [type.googleapis.com/mediapipe.ImageTransformationCalculatorOptions] { - output_width: 192 - output_height: 192 - scale_mode: FIT - } - } -} - -# Converts the transformed input image on CPU into an image tensor stored as a -# TfLiteTensor. -node { - calculator: "TfLiteConverterCalculator" - input_stream: "IMAGE:transformed_input_video_cpu" - output_stream: "TENSORS:image_tensor" -} - -# Runs a TensorFlow Lite model on CPU that takes an image tensor and outputs a -# vector of tensors representing, for instance, detection boxes/keypoints and -# scores. -node { - calculator: "TfLiteInferenceCalculator" - input_stream: "TENSORS:image_tensor" - output_stream: "TENSORS:detection_tensors" - node_options: { - [type.googleapis.com/mediapipe.TfLiteInferenceCalculatorOptions] { - model_path: "mediapipe/modules/face_detection/face_detection_back.tflite" - } - } -} - -# Generates a single side packet containing a vector of SSD anchors based on -# the specification in the options. -node { - calculator: "SsdAnchorsCalculator" - output_side_packet: "anchors" - node_options: { - [type.googleapis.com/mediapipe.SsdAnchorsCalculatorOptions] { - num_layers: 1 - min_scale: 0.1484375 - max_scale: 0.75 - input_size_height: 192 - input_size_width: 192 - anchor_offset_x: 0.5 - anchor_offset_y: 0.5 - strides: 4 - aspect_ratios: 1.0 - fixed_anchor_size: true - interpolated_scale_aspect_ratio: 0.0 - } - } -} - -# Decodes the detection tensors generated by the TensorFlow Lite model, based on -# the SSD anchors and the specification in the options, into a vector of -# detections. Each detection describes a detected object. -node { - calculator: "TfLiteTensorsToDetectionsCalculator" - input_stream: "TENSORS:detection_tensors" - input_side_packet: "ANCHORS:anchors" - output_stream: "DETECTIONS:detections" - node_options: { - [type.googleapis.com/mediapipe.TfLiteTensorsToDetectionsCalculatorOptions] { - num_classes: 1 - num_boxes: 2304 - num_coords: 16 - box_coord_offset: 0 - keypoint_coord_offset: 4 - num_keypoints: 6 - num_values_per_keypoint: 2 - sigmoid_score: true - score_clipping_thresh: 100.0 - reverse_output_order: true - x_scale: 192.0 - y_scale: 192.0 - h_scale: 192.0 - w_scale: 192.0 - min_score_thresh: 0.6 - } - } -} - -# Performs non-max suppression to remove excessive detections. -node { - calculator: "NonMaxSuppressionCalculator" - input_stream: "detections" - output_stream: "filtered_detections" - node_options: { - [type.googleapis.com/mediapipe.NonMaxSuppressionCalculatorOptions] { - min_suppression_threshold: 0.3 - overlap_type: INTERSECTION_OVER_UNION - algorithm: WEIGHTED - return_empty_detections: true - } - } -} - -# Adjusts detection locations (already normalized to [0.f, 1.f]) on the -# letterboxed image (after image transformation with the FIT scale mode) to the -# corresponding locations on the same image with the letterbox removed (the -# input image to the graph before image transformation). -node { - calculator: "DetectionLetterboxRemovalCalculator" - input_stream: "DETECTIONS:filtered_detections" - input_stream: "LETTERBOX_PADDING:letterbox_padding" - output_stream: "DETECTIONS:output_detections" -} - -# Converts the detections to drawing primitives for annotation overlay. -node { - calculator: "DetectionsToRenderDataCalculator" - input_stream: "DETECTIONS:output_detections" - output_stream: "RENDER_DATA:render_data" - node_options: { - [type.googleapis.com/mediapipe.DetectionsToRenderDataCalculatorOptions] { - thickness: 4.0 - color { r: 255 g: 0 b: 0 } - } - } -} - -# Draws annotations and overlays them on top of the input images. -node { - calculator: "AnnotationOverlayCalculator" - input_stream: "IMAGE:throttled_input_video" - input_stream: "render_data" - output_stream: "IMAGE:output_video" -} - diff --git a/mediapipe/graphs/face_detection/face_detection_back_mobile_gpu.pbtxt b/mediapipe/graphs/face_detection/face_detection_back_mobile_gpu.pbtxt deleted file mode 100644 index c69bf50a..00000000 --- a/mediapipe/graphs/face_detection/face_detection_back_mobile_gpu.pbtxt +++ /dev/null @@ -1,169 +0,0 @@ -# MediaPipe graph that performs face detection with TensorFlow Lite on GPU. -# Used in the examples in -# mediapipie/examples/android/src/java/com/mediapipe/apps/facedetectiongpu and -# mediapipie/examples/ios/facedetectiongpu. - -# Images on GPU coming into and out of the graph. -input_stream: "input_video" -output_stream: "output_video" - -# Throttles the images flowing downstream for flow control. It passes through -# the very first incoming image unaltered, and waits for -# TfLiteTensorsToDetectionsCalculator downstream in the graph to finish -# generating the corresponding detections before it passes through another -# image. All images that come in while waiting are dropped, limiting the number -# of in-flight images between this calculator and -# TfLiteTensorsToDetectionsCalculator to 1. This prevents the nodes in between -# from queuing up incoming images and data excessively, which leads to increased -# latency and memory usage, unwanted in real-time mobile applications. It also -# eliminates unnecessarily computation, e.g., a transformed image produced by -# ImageTransformationCalculator may get dropped downstream if the subsequent -# TfLiteConverterCalculator or TfLiteInferenceCalculator is still busy -# processing previous inputs. -node { - calculator: "FlowLimiterCalculator" - input_stream: "input_video" - input_stream: "FINISHED:detections" - input_stream_info: { - tag_index: "FINISHED" - back_edge: true - } - output_stream: "throttled_input_video" -} - -# Transforms the input image on GPU to a 128x128 image. To scale the input -# image, the scale_mode option is set to FIT to preserve the aspect ratio, -# resulting in potential letterboxing in the transformed image. -node: { - calculator: "ImageTransformationCalculator" - input_stream: "IMAGE_GPU:throttled_input_video" - output_stream: "IMAGE_GPU:transformed_input_video" - output_stream: "LETTERBOX_PADDING:letterbox_padding" - node_options: { - [type.googleapis.com/mediapipe.ImageTransformationCalculatorOptions] { - output_width: 192 - output_height: 192 - scale_mode: FIT - } - } -} - -# Converts the transformed input image on GPU into an image tensor stored as a -# TfLiteTensor. -node { - calculator: "TfLiteConverterCalculator" - input_stream: "IMAGE_GPU:transformed_input_video" - output_stream: "TENSORS_GPU:image_tensor" -} - -# Runs a TensorFlow Lite model on GPU that takes an image tensor and outputs a -# vector of tensors representing, for instance, detection boxes/keypoints and -# scores. -node { - calculator: "TfLiteInferenceCalculator" - input_stream: "TENSORS_GPU:image_tensor" - output_stream: "TENSORS_GPU:detection_tensors" - node_options: { - [type.googleapis.com/mediapipe.TfLiteInferenceCalculatorOptions] { - model_path: "mediapipe/modules/face_detection/face_detection_back.tflite" - } - } -} - -# Generates a single side packet containing a vector of SSD anchors based on -# the specification in the options. -node { - calculator: "SsdAnchorsCalculator" - output_side_packet: "anchors" - node_options: { - [type.googleapis.com/mediapipe.SsdAnchorsCalculatorOptions] { - num_layers: 1 - min_scale: 0.1484375 - max_scale: 0.75 - input_size_height: 192 - input_size_width: 192 - anchor_offset_x: 0.5 - anchor_offset_y: 0.5 - strides: 4 - aspect_ratios: 1.0 - fixed_anchor_size: true - interpolated_scale_aspect_ratio: 0.0 - } - } -} - -# Decodes the detection tensors generated by the TensorFlow Lite model, based on -# the SSD anchors and the specification in the options, into a vector of -# detections. Each detection describes a detected object. -node { - calculator: "TfLiteTensorsToDetectionsCalculator" - input_stream: "TENSORS_GPU:detection_tensors" - input_side_packet: "ANCHORS:anchors" - output_stream: "DETECTIONS:detections" - node_options: { - [type.googleapis.com/mediapipe.TfLiteTensorsToDetectionsCalculatorOptions] { - num_classes: 1 - num_boxes: 2304 - num_coords: 16 - box_coord_offset: 0 - keypoint_coord_offset: 4 - num_keypoints: 6 - num_values_per_keypoint: 2 - sigmoid_score: true - score_clipping_thresh: 100.0 - reverse_output_order: true - x_scale: 192.0 - y_scale: 192.0 - h_scale: 192.0 - w_scale: 192.0 - min_score_thresh: 0.6 - } - } -} - -# Performs non-max suppression to remove excessive detections. -node { - calculator: "NonMaxSuppressionCalculator" - input_stream: "detections" - output_stream: "filtered_detections" - node_options: { - [type.googleapis.com/mediapipe.NonMaxSuppressionCalculatorOptions] { - min_suppression_threshold: 0.3 - overlap_type: INTERSECTION_OVER_UNION - algorithm: WEIGHTED - return_empty_detections: true - } - } -} - -# Adjusts detection locations (already normalized to [0.f, 1.f]) on the -# letterboxed image (after image transformation with the FIT scale mode) to the -# corresponding locations on the same image with the letterbox removed (the -# input image to the graph before image transformation). -node { - calculator: "DetectionLetterboxRemovalCalculator" - input_stream: "DETECTIONS:filtered_detections" - input_stream: "LETTERBOX_PADDING:letterbox_padding" - output_stream: "DETECTIONS:output_detections" -} - -# Converts the detections to drawing primitives for annotation overlay. -node { - calculator: "DetectionsToRenderDataCalculator" - input_stream: "DETECTIONS:output_detections" - output_stream: "RENDER_DATA:render_data" - node_options: { - [type.googleapis.com/mediapipe.DetectionsToRenderDataCalculatorOptions] { - thickness: 4.0 - color { r: 255 g: 0 b: 0 } - } - } -} - -# Draws annotations and overlays them on top of the input images. -node { - calculator: "AnnotationOverlayCalculator" - input_stream: "IMAGE_GPU:throttled_input_video" - input_stream: "render_data" - output_stream: "IMAGE_GPU:output_video" -} diff --git a/mediapipe/graphs/face_detection/face_detection_desktop_live.pbtxt b/mediapipe/graphs/face_detection/face_detection_desktop_live.pbtxt index 06478a7a..023bea97 100644 --- a/mediapipe/graphs/face_detection/face_detection_desktop_live.pbtxt +++ b/mediapipe/graphs/face_detection/face_detection_desktop_live.pbtxt @@ -31,7 +31,7 @@ node { # Subgraph that detects faces. node { - calculator: "FaceDetectionFrontCpu" + calculator: "FaceDetectionShortRangeCpu" input_stream: "IMAGE:throttled_input_video" output_stream: "DETECTIONS:face_detections" } diff --git a/mediapipe/graphs/face_detection/face_detection_full_range_desktop_live.pbtxt b/mediapipe/graphs/face_detection/face_detection_full_range_desktop_live.pbtxt new file mode 100644 index 00000000..0fdb6b91 --- /dev/null +++ b/mediapipe/graphs/face_detection/face_detection_full_range_desktop_live.pbtxt @@ -0,0 +1,60 @@ +# MediaPipe graph that performs face detection with TensorFlow Lite on CPU. +# Used in the examples in +# mediapipe/examples/desktop/face_detection:face_detection_cpu. + +# Images on GPU coming into and out of the graph. +input_stream: "input_video" +output_stream: "output_video" + +# Throttles the images flowing downstream for flow control. It passes through +# the very first incoming image unaltered, and waits for +# TfLiteTensorsToDetectionsCalculator downstream in the graph to finish +# generating the corresponding detections before it passes through another +# image. All images that come in while waiting are dropped, limiting the number +# of in-flight images between this calculator and +# TfLiteTensorsToDetectionsCalculator to 1. This prevents the nodes in between +# from queuing up incoming images and data excessively, which leads to increased +# latency and memory usage, unwanted in real-time mobile applications. It also +# eliminates unnecessarily computation, e.g., a transformed image produced by +# ImageTransformationCalculator may get dropped downstream if the subsequent +# TfLiteConverterCalculator or TfLiteInferenceCalculator is still busy +# processing previous inputs. +node { + calculator: "FlowLimiterCalculator" + input_stream: "input_video" + input_stream: "FINISHED:detections" + input_stream_info: { + tag_index: "FINISHED" + back_edge: true + } + output_stream: "throttled_input_video" +} + +# Detects faces. +node { + calculator: "FaceDetectionFullRangeCpu" + input_stream: "IMAGE:throttled_input_video" + output_stream: "DETECTIONS:detections" +} + +# Converts the detections to drawing primitives for annotation overlay. +node { + calculator: "DetectionsToRenderDataCalculator" + input_stream: "DETECTIONS:detections" + output_stream: "RENDER_DATA:render_data" + node_options: { + [type.googleapis.com/mediapipe.DetectionsToRenderDataCalculatorOptions] { + thickness: 4.0 + color { r: 255 g: 0 b: 0 } + } + } +} + +# Draws annotations and overlays them on top of the input images. +node { + calculator: "AnnotationOverlayCalculator" + input_stream: "IMAGE:throttled_input_video" + input_stream: "render_data" + output_stream: "IMAGE:output_video" +} + diff --git a/mediapipe/graphs/face_detection/face_detection_full_range_mobile_gpu.pbtxt b/mediapipe/graphs/face_detection/face_detection_full_range_mobile_gpu.pbtxt new file mode 100644 index 00000000..8224543f --- /dev/null +++ b/mediapipe/graphs/face_detection/face_detection_full_range_mobile_gpu.pbtxt @@ -0,0 +1,60 @@ +# MediaPipe graph that performs face detection with TensorFlow Lite on GPU. +# Used in the examples in +# mediapipie/examples/android/src/java/com/mediapipe/apps/facedetectiongpu and +# mediapipie/examples/ios/facedetectiongpu. + +# Images on GPU coming into and out of the graph. +input_stream: "input_video" +output_stream: "output_video" + +# Throttles the images flowing downstream for flow control. It passes through +# the very first incoming image unaltered, and waits for +# TfLiteTensorsToDetectionsCalculator downstream in the graph to finish +# generating the corresponding detections before it passes through another +# image. All images that come in while waiting are dropped, limiting the number +# of in-flight images between this calculator and +# TfLiteTensorsToDetectionsCalculator to 1. This prevents the nodes in between +# from queuing up incoming images and data excessively, which leads to increased +# latency and memory usage, unwanted in real-time mobile applications. It also +# eliminates unnecessarily computation, e.g., a transformed image produced by +# ImageTransformationCalculator may get dropped downstream if the subsequent +# TfLiteConverterCalculator or TfLiteInferenceCalculator is still busy +# processing previous inputs. +node { + calculator: "FlowLimiterCalculator" + input_stream: "input_video" + input_stream: "FINISHED:output_video" + input_stream_info: { + tag_index: "FINISHED" + back_edge: true + } + output_stream: "throttled_input_video" +} + +# Detects faces. +node { + calculator: "FaceDetectionFullRangeGpu" + input_stream: "IMAGE:throttled_input_video" + output_stream: "DETECTIONS:detections" +} + +# Converts the detections to drawing primitives for annotation overlay. +node { + calculator: "DetectionsToRenderDataCalculator" + input_stream: "DETECTIONS:detections" + output_stream: "RENDER_DATA:render_data" + node_options: { + [type.googleapis.com/mediapipe.DetectionsToRenderDataCalculatorOptions] { + thickness: 4.0 + color { r: 255 g: 0 b: 0 } + } + } +} + +# Draws annotations and overlays them on top of the input images. +node { + calculator: "AnnotationOverlayCalculator" + input_stream: "IMAGE_GPU:throttled_input_video" + input_stream: "render_data" + output_stream: "IMAGE_GPU:output_video" +} diff --git a/mediapipe/graphs/face_detection/face_detection_mobile_cpu.pbtxt b/mediapipe/graphs/face_detection/face_detection_mobile_cpu.pbtxt index ac9c667a..681d2dbb 100644 --- a/mediapipe/graphs/face_detection/face_detection_mobile_cpu.pbtxt +++ b/mediapipe/graphs/face_detection/face_detection_mobile_cpu.pbtxt @@ -41,7 +41,7 @@ node: { # Subgraph that detects faces. node { - calculator: "FaceDetectionFrontCpu" + calculator: "FaceDetectionShortRangeCpu" input_stream: "IMAGE:input_video_cpu" output_stream: "DETECTIONS:face_detections" } diff --git a/mediapipe/graphs/face_detection/face_detection_mobile_gpu.pbtxt b/mediapipe/graphs/face_detection/face_detection_mobile_gpu.pbtxt index 904246a5..d235d1cc 100644 --- a/mediapipe/graphs/face_detection/face_detection_mobile_gpu.pbtxt +++ b/mediapipe/graphs/face_detection/face_detection_mobile_gpu.pbtxt @@ -31,7 +31,7 @@ node { # Subgraph that detects faces. node { - calculator: "FaceDetectionFrontGpu" + calculator: "FaceDetectionShortRangeGpu" input_stream: "IMAGE:throttled_input_video" output_stream: "DETECTIONS:face_detections" } diff --git a/mediapipe/graphs/face_effect/subgraphs/BUILD b/mediapipe/graphs/face_effect/subgraphs/BUILD index c38008e8..0b23ad5d 100644 --- a/mediapipe/graphs/face_effect/subgraphs/BUILD +++ b/mediapipe/graphs/face_effect/subgraphs/BUILD @@ -39,7 +39,7 @@ mediapipe_simple_subgraph( "//mediapipe/calculators/core:concatenate_detection_vector_calculator", "//mediapipe/calculators/core:split_vector_calculator", "//mediapipe/calculators/image:image_properties_calculator", - "//mediapipe/modules/face_detection:face_detection_front_gpu", + "//mediapipe/modules/face_detection:face_detection_short_range_gpu", "//mediapipe/modules/face_geometry:face_geometry_from_detection", ], ) diff --git a/mediapipe/graphs/face_effect/subgraphs/single_face_geometry_from_detection_gpu.pbtxt b/mediapipe/graphs/face_effect/subgraphs/single_face_geometry_from_detection_gpu.pbtxt index 58210758..bce72c1c 100644 --- a/mediapipe/graphs/face_effect/subgraphs/single_face_geometry_from_detection_gpu.pbtxt +++ b/mediapipe/graphs/face_effect/subgraphs/single_face_geometry_from_detection_gpu.pbtxt @@ -24,7 +24,7 @@ output_stream: "MULTI_FACE_GEOMETRY:multi_face_geometry" # Subgraph that detects faces and corresponding landmarks using the face # detection pipeline. node { - calculator: "FaceDetectionFrontGpu" + calculator: "FaceDetectionShortRangeGpu" input_stream: "IMAGE:input_image" output_stream: "DETECTIONS:multi_face_detection" } diff --git a/mediapipe/graphs/face_mesh/subgraphs/BUILD b/mediapipe/graphs/face_mesh/subgraphs/BUILD index a94b47a8..fbb946df 100644 --- a/mediapipe/graphs/face_mesh/subgraphs/BUILD +++ b/mediapipe/graphs/face_mesh/subgraphs/BUILD @@ -24,7 +24,7 @@ package(default_visibility = ["//visibility:public"]) cc_library( name = "renderer_calculators", deps = [ - "//mediapipe/calculators/core:split_normalized_landmark_list_calculator", + "//mediapipe/calculators/core:split_landmarks_calculator", "//mediapipe/calculators/util:annotation_overlay_calculator", "//mediapipe/calculators/util:detections_to_render_data_calculator", "//mediapipe/calculators/util:landmarks_to_render_data_calculator", diff --git a/mediapipe/graphs/holistic_tracking/BUILD b/mediapipe/graphs/holistic_tracking/BUILD index 14290e32..986cf9f3 100644 --- a/mediapipe/graphs/holistic_tracking/BUILD +++ b/mediapipe/graphs/holistic_tracking/BUILD @@ -30,7 +30,7 @@ mediapipe_simple_subgraph( "//mediapipe/calculators/core:concatenate_normalized_landmark_list_calculator", "//mediapipe/calculators/core:concatenate_vector_calculator", "//mediapipe/calculators/core:merge_calculator", - "//mediapipe/calculators/core:split_normalized_landmark_list_calculator", + "//mediapipe/calculators/core:split_landmarks_calculator", "//mediapipe/calculators/core:split_vector_calculator", "//mediapipe/calculators/util:detections_to_render_data_calculator", "//mediapipe/calculators/util:landmarks_to_render_data_calculator", diff --git a/mediapipe/graphs/iris_tracking/subgraphs/BUILD b/mediapipe/graphs/iris_tracking/subgraphs/BUILD index a6dd9e51..d37c5509 100644 --- a/mediapipe/graphs/iris_tracking/subgraphs/BUILD +++ b/mediapipe/graphs/iris_tracking/subgraphs/BUILD @@ -26,7 +26,7 @@ cc_library( deps = [ "//mediapipe/calculators/core:concatenate_normalized_landmark_list_calculator", "//mediapipe/calculators/core:concatenate_vector_calculator", - "//mediapipe/calculators/core:split_normalized_landmark_list_calculator", + "//mediapipe/calculators/core:split_landmarks_calculator", "//mediapipe/calculators/util:annotation_overlay_calculator", "//mediapipe/calculators/util:detection_label_id_to_text_calculator", "//mediapipe/calculators/util:detections_to_render_data_calculator", diff --git a/mediapipe/graphs/pose_tracking/subgraphs/BUILD b/mediapipe/graphs/pose_tracking/subgraphs/BUILD index bb089feb..176b3c03 100644 --- a/mediapipe/graphs/pose_tracking/subgraphs/BUILD +++ b/mediapipe/graphs/pose_tracking/subgraphs/BUILD @@ -26,7 +26,7 @@ mediapipe_simple_subgraph( graph = "pose_renderer_gpu.pbtxt", register_as = "PoseRendererGpu", deps = [ - "//mediapipe/calculators/core:split_normalized_landmark_list_calculator", + "//mediapipe/calculators/core:split_landmarks_calculator", "//mediapipe/calculators/util:annotation_overlay_calculator", "//mediapipe/calculators/util:detections_to_render_data_calculator", "//mediapipe/calculators/util:landmarks_to_render_data_calculator", @@ -40,7 +40,7 @@ mediapipe_simple_subgraph( graph = "pose_renderer_cpu.pbtxt", register_as = "PoseRendererCpu", deps = [ - "//mediapipe/calculators/core:split_normalized_landmark_list_calculator", + "//mediapipe/calculators/core:split_landmarks_calculator", "//mediapipe/calculators/util:annotation_overlay_calculator", "//mediapipe/calculators/util:detections_to_render_data_calculator", "//mediapipe/calculators/util:landmarks_to_render_data_calculator", diff --git a/mediapipe/java/com/google/mediapipe/components/GlSurfaceViewRenderer.java b/mediapipe/java/com/google/mediapipe/components/GlSurfaceViewRenderer.java index 694ffc5d..a3d6a87f 100644 --- a/mediapipe/java/com/google/mediapipe/components/GlSurfaceViewRenderer.java +++ b/mediapipe/java/com/google/mediapipe/components/GlSurfaceViewRenderer.java @@ -91,15 +91,15 @@ public class GlSurfaceViewRenderer implements GLSurfaceView.Renderer { GLES20.glViewport(0, 0, width, height); } - @Override - public void onDrawFrame(GL10 gl) { + /** Renders the frame. Note that the {@link #flush} method must be called afterwards. */ + protected TextureFrame renderFrame() { TextureFrame frame = nextFrame.getAndSet(null); GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT); ShaderUtil.checkGlError("glClear"); if (surfaceTexture == null && frame == null) { - return; + return null; } GLES20.glActiveTexture(GLES20.GL_TEXTURE0); @@ -161,14 +161,28 @@ public class GlSurfaceViewRenderer implements GLSurfaceView.Renderer { GLES20.glBindTexture(textureTarget, 0); ShaderUtil.checkGlError("unbind surfaceTexture"); - // We must flush before releasing the frame. - GLES20.glFlush(); + return frame; + } + /** + * Calls {@link #GLES20.glFlush} and releases the texture frame. Should be invoked after the + * {@link #renderFrame} method is called. + * + * @param frame the {@link TextureFrame} to be released after {@link #GLES20.glFlush}. + */ + protected void flush(TextureFrame frame) { + GLES20.glFlush(); if (frame != null) { frame.release(); } } + @Override + public void onDrawFrame(GL10 gl) { + TextureFrame frame = renderFrame(); + flush(frame); + } + public void setTextureTarget(int target) { if (program != 0) { throw new IllegalStateException( diff --git a/mediapipe/java/com/google/mediapipe/framework/AndroidPacketCreator.java b/mediapipe/java/com/google/mediapipe/framework/AndroidPacketCreator.java index 5ddeb98c..69c0ebeb 100644 --- a/mediapipe/java/com/google/mediapipe/framework/AndroidPacketCreator.java +++ b/mediapipe/java/com/google/mediapipe/framework/AndroidPacketCreator.java @@ -16,7 +16,6 @@ package com.google.mediapipe.framework; import android.graphics.Bitmap; import java.nio.ByteBuffer; -import java.util.List; // TODO: use Preconditions in this file. /** diff --git a/mediapipe/java/com/google/mediapipe/framework/GraphGlSyncToken.java b/mediapipe/java/com/google/mediapipe/framework/GraphGlSyncToken.java index c141a95a..2b48737a 100644 --- a/mediapipe/java/com/google/mediapipe/framework/GraphGlSyncToken.java +++ b/mediapipe/java/com/google/mediapipe/framework/GraphGlSyncToken.java @@ -19,7 +19,7 @@ package com.google.mediapipe.framework; * has reached the specified point in the sequence of commands it is executing. This can be * necessary when working with multiple GL contexts. */ -final class GraphGlSyncToken implements GlSyncToken { +public final class GraphGlSyncToken implements GlSyncToken { private long token; @Override @@ -44,7 +44,7 @@ final class GraphGlSyncToken implements GlSyncToken { } } - GraphGlSyncToken(long token) { + public GraphGlSyncToken(long token) { this.token = token; } diff --git a/mediapipe/java/com/google/mediapipe/framework/jni/BUILD b/mediapipe/java/com/google/mediapipe/framework/jni/BUILD index e16f140b..650671d6 100644 --- a/mediapipe/java/com/google/mediapipe/framework/jni/BUILD +++ b/mediapipe/java/com/google/mediapipe/framework/jni/BUILD @@ -73,6 +73,7 @@ cc_library( ], "//mediapipe/gpu:disable_gpu": [], }), + features = ["-no_undefined"], linkopts = select({ "//conditions:default": [], "//mediapipe:android": [ diff --git a/mediapipe/java/com/google/mediapipe/framework/jni/graph.cc b/mediapipe/java/com/google/mediapipe/framework/jni/graph.cc index e24df24f..f02cb83d 100644 --- a/mediapipe/java/com/google/mediapipe/framework/jni/graph.cc +++ b/mediapipe/java/com/google/mediapipe/framework/jni/graph.cc @@ -583,9 +583,9 @@ absl::Status Graph::SetParentGlContext(int64 java_gl_context) { #if MEDIAPIPE_DISABLE_GPU LOG(FATAL) << "GPU support has been disabled in this build!"; #else - gpu_resources_ = mediapipe::GpuResources::Create( - reinterpret_cast(java_gl_context)) - .value(); + ASSIGN_OR_RETURN(gpu_resources_, + mediapipe::GpuResources::Create( + reinterpret_cast(java_gl_context))); #endif // MEDIAPIPE_DISABLE_GPU return absl::OkStatus(); } diff --git a/mediapipe/java/com/google/mediapipe/mediapipe_aar.bzl b/mediapipe/java/com/google/mediapipe/mediapipe_aar.bzl index a0c5a503..b4ed08f8 100644 --- a/mediapipe/java/com/google/mediapipe/mediapipe_aar.bzl +++ b/mediapipe/java/com/google/mediapipe/mediapipe_aar.bzl @@ -46,6 +46,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_binary", "android def mediapipe_aar( name, srcs = [], + gen_libmediapipe = True, calculators = [], assets = [], assets_dir = ""): @@ -54,12 +55,14 @@ def mediapipe_aar( Args: name: the name of the aar. srcs: the additional java source code to be added into the android library. + gen_libmediapipe: whether to generate libmediapipe_jni.so. Default to True. calculators: the calculator libraries to be compiled into the jni library. assets: additional assets to be included into the archive. assets_dir: path where the assets will the packaged. """ _mediapipe_jni( name = name + "_jni", + gen_libmediapipe = gen_libmediapipe, calculators = calculators, ) @@ -67,6 +70,22 @@ def mediapipe_aar( name = name + "_proto", ) + native.genrule( + name = name + "_aar_manifest_generator", + outs = ["AndroidManifest.xml"], + cmd = """ +cat > $(OUTS) < + + + +EOF +""", + ) + android_library( name = name + "_android_lib", srcs = srcs + [ @@ -84,7 +103,6 @@ def mediapipe_aar( proguard_specs = ["//mediapipe/java/com/google/mediapipe/framework:proguard.pgcfg"], deps = [ ":" + name + "_jni_cc_lib", - ":" + name + "_jni_opencv_cc_lib", "//mediapipe/framework:calculator_java_proto_lite", "//mediapipe/framework:calculator_profile_java_proto_lite", "//mediapipe/framework:calculator_options_java_proto_lite", @@ -94,6 +112,10 @@ def mediapipe_aar( "//mediapipe/framework:status_handler_java_proto_lite", "//mediapipe/framework:stream_handler_java_proto_lite", "//mediapipe/framework/tool:calculator_graph_template_java_proto_lite", + "//mediapipe/java/com/google/mediapipe/components:android_components", + "//mediapipe/java/com/google/mediapipe/components:android_camerax_helper", + "//mediapipe/java/com/google/mediapipe/framework:android_framework", + "//mediapipe/java/com/google/mediapipe/glutil", "//third_party:androidx_annotation", "//third_party:androidx_appcompat", "//third_party:androidx_core", @@ -108,7 +130,10 @@ def mediapipe_aar( "@maven//:com_google_flogger_flogger_system_backend", "@maven//:com_google_guava_guava", "@maven//:androidx_lifecycle_lifecycle_common", - ], + ] + select({ + "//conditions:default": [":" + name + "_jni_opencv_cc_lib"], + "//mediapipe/framework/port:disable_opencv": [], + }), assets = assets, assets_dir = assets_dir, ) @@ -121,22 +146,6 @@ def _mediapipe_proto(name): Args: name: the name of the target. """ - native.genrule( - name = name + "_aar_manifest_generator", - outs = ["AndroidManifest.xml"], - cmd = """ -cat > $(OUTS) < - - - - -""", - ) - _proto_java_src_generator( name = "calculator_proto", proto_src = "mediapipe/framework/calculator.proto", @@ -204,21 +213,23 @@ def _proto_java_src_generator(name, proto_src, java_lite_out, srcs = []): ], ) -def _mediapipe_jni(name, calculators = []): +def _mediapipe_jni(name, gen_libmediapipe, calculators = []): """Generates MediaPipe jni library. Args: name: the name of the target. + gen_libmediapipe: whether to generate libmediapipe_jni.so. Default to True. calculators: the calculator libraries to be compiled into the jni library. """ - native.cc_binary( - name = "libmediapipe_jni.so", - linkshared = 1, - linkstatic = 1, - deps = [ - "//mediapipe/java/com/google/mediapipe/framework/jni:mediapipe_framework_jni", - ] + calculators, - ) + if gen_libmediapipe: + native.cc_binary( + name = "libmediapipe_jni.so", + linkshared = 1, + linkstatic = 1, + deps = [ + "//mediapipe/java/com/google/mediapipe/framework/jni:mediapipe_framework_jni", + ] + calculators, + ) native.cc_library( name = name + "_cc_lib", diff --git a/mediapipe/java/com/google/mediapipe/solutionbase/BUILD b/mediapipe/java/com/google/mediapipe/solutionbase/BUILD index a3acad5d..e131061f 100644 --- a/mediapipe/java/com/google/mediapipe/solutionbase/BUILD +++ b/mediapipe/java/com/google/mediapipe/solutionbase/BUILD @@ -22,6 +22,9 @@ android_library( ["*.java"], exclude = [ "CameraInput.java", + "ResultGlRenderer.java", + "SolutionGlSurfaceView.java", + "SolutionGlSurfaceViewRenderer.java", ], ), visibility = ["//visibility:public"], @@ -29,6 +32,7 @@ android_library( "//mediapipe/java/com/google/mediapipe/framework:android_framework", "//mediapipe/java/com/google/mediapipe/glutil", "//third_party:autovalue", + "@com_google_protobuf//:protobuf_javalite", "@maven//:com_google_code_findbugs_jsr305", "@maven//:com_google_guava_guava", ], @@ -46,6 +50,23 @@ android_library( ], ) +android_library( + name = "solution_rendering", + srcs = [ + "ResultGlRenderer.java", + "SolutionGlSurfaceView.java", + "SolutionGlSurfaceViewRenderer.java", + ], + visibility = ["//visibility:public"], + deps = [ + ":solution_base", + "//mediapipe/java/com/google/mediapipe/components:android_components", + "//mediapipe/java/com/google/mediapipe/framework:android_framework", + "//mediapipe/java/com/google/mediapipe/glutil", + "@maven//:com_google_guava_guava", + ], +) + # Native dependencies of all MediaPipe solutions. cc_binary( name = "libmediapipe_jni.so", @@ -65,3 +86,11 @@ cc_library( visibility = ["//visibility:public"], alwayslink = 1, ) + +load("//mediapipe/java/com/google/mediapipe:mediapipe_aar.bzl", "mediapipe_aar") + +mediapipe_aar( + name = "solution_core", + srcs = glob(["*.java"]), + gen_libmediapipe = False, +) diff --git a/mediapipe/java/com/google/mediapipe/solutionbase/ImageSolutionResult.java b/mediapipe/java/com/google/mediapipe/solutionbase/ImageSolutionResult.java index 9e8cc11a..0fcabed7 100644 --- a/mediapipe/java/com/google/mediapipe/solutionbase/ImageSolutionResult.java +++ b/mediapipe/java/com/google/mediapipe/solutionbase/ImageSolutionResult.java @@ -39,6 +39,9 @@ public class ImageSolutionResult implements SolutionResult { // Returns the corresponding input image as a {@link Bitmap}. public Bitmap inputBitmap() { + if (imagePacket == null) { + return null; + } if (cachedBitmap != null) { return cachedBitmap; } @@ -49,6 +52,9 @@ public class ImageSolutionResult implements SolutionResult { // Returns the corresponding input image as a {@link TextureFrame}. The caller must release the // acquired {@link TextureFrame} after using. public TextureFrame acquireTextureFrame() { + if (imagePacket == null) { + return null; + } return PacketGetter.getTextureFrame(imagePacket); } diff --git a/mediapipe/java/com/google/mediapipe/solutionbase/ResultGlRenderer.java b/mediapipe/java/com/google/mediapipe/solutionbase/ResultGlRenderer.java new file mode 100644 index 00000000..1cabb706 --- /dev/null +++ b/mediapipe/java/com/google/mediapipe/solutionbase/ResultGlRenderer.java @@ -0,0 +1,25 @@ +// Copyright 2021 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.mediapipe.solutionbase; + +/** Interface for the customizable MediaPipe solution result OpenGL renderer. */ +public interface ResultGlRenderer { + + /** Sets up OpenGL rendering when the surface is created or recreated. */ + void setupRendering(); + + /** Renders the solution result. */ + void renderResult(T result); +} diff --git a/mediapipe/java/com/google/mediapipe/solutionbase/SolutionBase.java b/mediapipe/java/com/google/mediapipe/solutionbase/SolutionBase.java index d4219456..2655c22a 100644 --- a/mediapipe/java/com/google/mediapipe/solutionbase/SolutionBase.java +++ b/mediapipe/java/com/google/mediapipe/solutionbase/SolutionBase.java @@ -45,9 +45,7 @@ public class SolutionBase { protected final AtomicBoolean solutionGraphStarted = new AtomicBoolean(false); static { - // Load all native libraries needed by the app. System.loadLibrary("mediapipe_jni"); - System.loadLibrary("opencv_java3"); } /** diff --git a/mediapipe/java/com/google/mediapipe/solutionbase/SolutionGlSurfaceView.java b/mediapipe/java/com/google/mediapipe/solutionbase/SolutionGlSurfaceView.java new file mode 100644 index 00000000..878064f5 --- /dev/null +++ b/mediapipe/java/com/google/mediapipe/solutionbase/SolutionGlSurfaceView.java @@ -0,0 +1,118 @@ +// Copyright 2021 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.mediapipe.solutionbase; + +import android.content.Context; +import android.opengl.GLES20; +import android.opengl.GLSurfaceView; +import android.util.Log; +import android.view.SurfaceHolder; +import android.view.View; +import com.google.mediapipe.glutil.EglManager; +import javax.microedition.khronos.egl.EGL10; +import javax.microedition.khronos.egl.EGLConfig; +import javax.microedition.khronos.egl.EGLContext; +import javax.microedition.khronos.egl.EGLDisplay; + +/** + * A simplified GlSurfaceView implementation for displaying MediaPipe Solution results. + * + *

Users need to provide a custom {@link ResultGlRenderer} via {@link + * setSolutionResultRenderer(ResultGlRenderer)} for rendering MediaPipe solution results. Setting + * the latest render data by calling {@link #setRenderData(ImageSolutionResult)} before invoking + * {@link #requestRender}. By default, the solution renderer renders the input images. Call {@link + * #setRenderInputImage(boolean)} to explicitly set whether the input images should be rendered or + * not. + */ +public class SolutionGlSurfaceView extends GLSurfaceView { + private static final String TAG = "SolutionGlSurfaceView"; + SolutionGlSurfaceViewRenderer renderer = new SolutionGlSurfaceViewRenderer<>(); + + /** + * Sets a user-defined {@link ResultGlRenderer} for rendering MediaPipe solution results. + * + * @param resultRenderer a {@link ResultGlRenderer}. + */ + public void setSolutionResultRenderer(ResultGlRenderer resultRenderer) { + renderer.setSolutionResultRenderer(resultRenderer); + } + + /** + * Sets the next textureframe and solution result to render. + * + * @param solutionResult a solution result object that contains the solution outputs and a + * textureframe. + */ + public void setRenderData(T solutionResult) { + renderer.setRenderData(solutionResult); + } + + /** Sets if the input image needs to be rendered. Default to true. */ + public void setRenderInputImage(boolean renderInputImage) { + renderer.setRenderInputImage(renderInputImage); + } + + /** Initializes SolutionGlSurfaceView with Android context, gl context, and gl version number. */ + public SolutionGlSurfaceView(Context context, EGLContext glContext, int glMajorVersion) { + super(context); + setEGLContextClientVersion(glMajorVersion); + getHolder().addCallback(new HolderCallbacks()); + setEGLContextFactory( + new GLSurfaceView.EGLContextFactory() { + @Override + public EGLContext createContext(EGL10 egl, EGLDisplay display, EGLConfig eglConfig) { + int[] contextAttrs = { + EglManager.EGL_CONTEXT_CLIENT_VERSION, glMajorVersion, EGL10.EGL_NONE + }; + return egl.eglCreateContext(display, eglConfig, glContext, contextAttrs); + } + + @Override + public void destroyContext(EGL10 egl, EGLDisplay display, EGLContext context) { + if (!egl.eglDestroyContext(display, context)) { + throw new RuntimeException("eglDestroyContext failed"); + } + } + }); + renderer.setTextureTarget(GLES20.GL_TEXTURE_2D); + super.setRenderer(renderer); + setRenderMode(GLSurfaceView.RENDERMODE_WHEN_DIRTY); + setVisibility(View.GONE); + } + + private class HolderCallbacks implements SurfaceHolder.Callback { + @Override + public void surfaceCreated(SurfaceHolder holder) { + Log.d(TAG, "main surfaceCreated"); + } + + @Override + public void surfaceChanged(SurfaceHolder holder, int format, int width, int height) { + Log.d( + TAG, + String.format( + "main surfaceChanged. width: %d height: %d glViewWidth: %d glViewHeight: %d", + width, + height, + SolutionGlSurfaceView.this.getWidth(), + SolutionGlSurfaceView.this.getHeight())); + } + + @Override + public void surfaceDestroyed(SurfaceHolder holder) { + Log.d(TAG, "main surfaceDestroyed"); + } + } +} diff --git a/mediapipe/java/com/google/mediapipe/solutionbase/SolutionGlSurfaceViewRenderer.java b/mediapipe/java/com/google/mediapipe/solutionbase/SolutionGlSurfaceViewRenderer.java new file mode 100644 index 00000000..11059c75 --- /dev/null +++ b/mediapipe/java/com/google/mediapipe/solutionbase/SolutionGlSurfaceViewRenderer.java @@ -0,0 +1,83 @@ +// Copyright 2021 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.mediapipe.solutionbase; + +import android.graphics.SurfaceTexture; +import com.google.mediapipe.components.GlSurfaceViewRenderer; +import com.google.mediapipe.framework.TextureFrame; +import java.util.concurrent.atomic.AtomicReference; +import javax.microedition.khronos.egl.EGLConfig; +import javax.microedition.khronos.opengles.GL10; + +/** + * MediaPipe Solution's GlSurfaceViewRenderer. + * + *

Users can provide a custom {@link ResultGlRenderer} for rendering MediaPipe solution results. + * For setting the latest solution result, call {@link #setRenderData(ImageSolutionResult)}. By + * default, the renderer renders the input images. Call {@link #setRenderInputImage(boolean)} to + * explicitly set whether the input images should be rendered or not. + */ +public class SolutionGlSurfaceViewRenderer + extends GlSurfaceViewRenderer { + private static final String TAG = "SolutionGlSurfaceViewRenderer"; + private boolean renderInputImage = true; + private final AtomicReference nextSolutionResult = new AtomicReference<>(); + private ResultGlRenderer resultGlRenderer; + + /** Sets if the input image needs to be rendered. Default to true. */ + public void setRenderInputImage(boolean renderInputImage) { + this.renderInputImage = renderInputImage; + } + + /** Sets a user-defined {@link ResultGlRenderer} for rendering MediaPipe solution results. */ + public void setSolutionResultRenderer(ResultGlRenderer resultGlRenderer) { + this.resultGlRenderer = resultGlRenderer; + } + + /** + * Sets the next textureframe and solution result to render. + * + * @param solutionResult a solution result object that contains the solution outputs and a + * textureframe. + */ + public void setRenderData(T solutionResult) { + setNextFrame(solutionResult.acquireTextureFrame()); + nextSolutionResult.getAndSet(solutionResult); + } + + @Override + public void onSurfaceCreated(GL10 gl, EGLConfig config) { + super.onSurfaceCreated(gl, config); + resultGlRenderer.setupRendering(); + } + + @Override + public void onDrawFrame(GL10 gl) { + TextureFrame frame = null; + if (renderInputImage) { + frame = renderFrame(); + } + if (nextSolutionResult != null) { + T solutionResult = nextSolutionResult.getAndSet(null); + resultGlRenderer.renderResult(solutionResult); + } + flush(frame); + } + + @Override + public void setSurfaceTexture(SurfaceTexture texture) { + throw new IllegalStateException("SurfaceTexture should not be used in MediaPipe Solution."); + } +} diff --git a/mediapipe/java/com/google/mediapipe/solutions/hands/AndroidManifest.xml b/mediapipe/java/com/google/mediapipe/solutions/hands/AndroidManifest.xml new file mode 100644 index 00000000..17853d5a --- /dev/null +++ b/mediapipe/java/com/google/mediapipe/solutions/hands/AndroidManifest.xml @@ -0,0 +1,8 @@ + + + + + + diff --git a/mediapipe/java/com/google/mediapipe/solutions/hands/BUILD b/mediapipe/java/com/google/mediapipe/solutions/hands/BUILD new file mode 100644 index 00000000..632dfd96 --- /dev/null +++ b/mediapipe/java/com/google/mediapipe/solutions/hands/BUILD @@ -0,0 +1,45 @@ +# Copyright 2021 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +licenses(["notice"]) + +android_library( + name = "hands", + srcs = [ + "HandLandmark.java", + "Hands.java", + "HandsOptions.java", + "HandsResult.java", + ], + assets = [ + "//mediapipe/modules/hand_landmark:hand_landmark_tracking_gpu_image.binarypb", + "//mediapipe/modules/hand_landmark:handedness.txt", + "//mediapipe/modules/hand_landmark:hand_landmark.tflite", + "//mediapipe/modules/palm_detection:palm_detection.tflite", + ], + assets_dir = "", + javacopts = ["-Acom.google.auto.value.AutoBuilderIsUnstable"], + manifest = ":AndroidManifest.xml", + visibility = ["//visibility:public"], + deps = [ + "//mediapipe/framework/formats:classification_java_proto_lite", + "//mediapipe/framework/formats:landmark_java_proto_lite", + "//mediapipe/java/com/google/mediapipe/framework:android_framework", + "//mediapipe/java/com/google/mediapipe/solutionbase:solution_base", + "//third_party:autovalue", + "@maven//:androidx_annotation_annotation", + "@maven//:com_google_code_findbugs_jsr305", + "@maven//:com_google_guava_guava", + ], +) diff --git a/mediapipe/java/com/google/mediapipe/solutions/hands/HandLandmark.java b/mediapipe/java/com/google/mediapipe/solutions/hands/HandLandmark.java new file mode 100644 index 00000000..79e485f0 --- /dev/null +++ b/mediapipe/java/com/google/mediapipe/solutions/hands/HandLandmark.java @@ -0,0 +1,72 @@ +// Copyright 2021 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.mediapipe.solutions.hands; + +import androidx.annotation.IntDef; + +/** The 21 hand landmarks. */ +public final class HandLandmark { + public static final int NUM_LANDMARKS = 21; + + public static final int WRIST = 0; + public static final int THUMB_CMC = 1; + public static final int THUMB_MCP = 2; + public static final int THUMB_DIP = 3; + public static final int THUMB_TIP = 4; + public static final int INDEX_FINGER_MCP = 5; + public static final int INDEX_FINGER_PIP = 6; + public static final int INDEX_FINGER_DIP = 7; + public static final int INDEX_FINGER_TIP = 8; + public static final int MIDDLE_FINGER_MCP = 9; + public static final int MIDDLE_FINGER_PIP = 10; + public static final int MIDDLE_FINGER_DIP = 11; + public static final int MIDDLE_FINGER_TIP = 12; + public static final int RING_FINGER_MCP = 13; + public static final int RING_FINGER_PIP = 14; + public static final int RING_FINGER_DIP = 15; + public static final int RING_FINGER_TIP = 16; + public static final int PINKY_MCP = 17; + public static final int PINKY_PIP = 18; + public static final int PINKY_DIP = 19; + public static final int PINKY_TIP = 20; + + /** Represents a hand landmark type. */ + @IntDef({ + WRIST, + THUMB_CMC, + THUMB_MCP, + THUMB_DIP, + THUMB_TIP, + INDEX_FINGER_MCP, + INDEX_FINGER_PIP, + INDEX_FINGER_DIP, + INDEX_FINGER_TIP, + MIDDLE_FINGER_MCP, + MIDDLE_FINGER_PIP, + MIDDLE_FINGER_DIP, + MIDDLE_FINGER_TIP, + RING_FINGER_MCP, + RING_FINGER_PIP, + RING_FINGER_DIP, + RING_FINGER_TIP, + PINKY_MCP, + PINKY_PIP, + PINKY_DIP, + PINKY_TIP, + }) + public @interface HandLandmarkType {} + + private HandLandmark() {} +} diff --git a/mediapipe/java/com/google/mediapipe/solutions/hands/Hands.java b/mediapipe/java/com/google/mediapipe/solutions/hands/Hands.java new file mode 100644 index 00000000..044d177d --- /dev/null +++ b/mediapipe/java/com/google/mediapipe/solutions/hands/Hands.java @@ -0,0 +1,132 @@ +// Copyright 2021 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.mediapipe.solutions.hands; + +import android.content.Context; +import com.google.common.collect.ImmutableList; +import com.google.mediapipe.formats.proto.LandmarkProto.NormalizedLandmark; +import com.google.mediapipe.formats.proto.LandmarkProto.NormalizedLandmarkList; +import com.google.mediapipe.formats.proto.ClassificationProto.Classification; +import com.google.mediapipe.framework.MediaPipeException; +import com.google.mediapipe.framework.Packet; +import com.google.mediapipe.solutionbase.ErrorListener; +import com.google.mediapipe.solutionbase.ImageSolutionBase; +import com.google.mediapipe.solutionbase.OutputHandler; +import com.google.mediapipe.solutionbase.ResultListener; +import com.google.mediapipe.solutionbase.SolutionInfo; +import java.util.HashMap; +import java.util.Map; +import javax.annotation.Nullable; + +/** + * MediaPipe Hands Solution API. + * + *

MediaPipe Hands processes a {@link TextureFrame} or a {@link Bitmap} and returns the hand + * landmarks and handedness (left v.s. right hand) of each detected hand. Please refer to + * https://solutions.mediapipe.dev/hands#android-solution-api for usage examples. + */ +public class Hands extends ImageSolutionBase { + private static final String TAG = "Hands"; + + private static final String NUM_HANDS = "num_hands"; + private static final String SOLUTION_GRAPH_NAME = "hand_landmark_tracking_gpu_image.binarypb"; + private static final String IMAGE_INPUT_STREAM = "image"; + private static final ImmutableList OUTPUT_STREAMS = + ImmutableList.of("multi_hand_landmarks", "multi_handedness", "image"); + private static final int LANDMARKS_INDEX = 0; + private static final int HANDEDNESS_INDEX = 1; + private static final int INPUT_IMAGE_INDEX = 2; + private final OutputHandler graphOutputHandler; + + /** + * Initializes MediaPipe Hands solution. + * + * @param context an Android {@link Context}. + * @param options the configuration options defined in {@link HandsOptions}. + */ + public Hands(Context context, HandsOptions options) { + graphOutputHandler = new OutputHandler<>(); + graphOutputHandler.setOutputConverter( + packets -> { + HandsResult.Builder handsResultBuilder = HandsResult.builder(); + try { + handsResultBuilder.setMultiHandLandmarks( + getProtoVector(packets.get(LANDMARKS_INDEX), NormalizedLandmarkList.parser())); + } catch (MediaPipeException e) { + throwException("Error occurs when getting MediaPipe hand landmarks. ", e); + } + try { + handsResultBuilder.setMultiHandedness( + getProtoVector(packets.get(HANDEDNESS_INDEX), Classification.parser())); + } catch (MediaPipeException e) { + throwException("Error occurs when getting MediaPipe handedness data. ", e); + } + return handsResultBuilder + .setImagePacket(packets.get(INPUT_IMAGE_INDEX)) + .setTimestamp( + staticImageMode ? Long.MIN_VALUE : packets.get(INPUT_IMAGE_INDEX).getTimestamp()) + .build(); + }); + + SolutionInfo solutionInfo = + SolutionInfo.builder() + .setBinaryGraphPath(SOLUTION_GRAPH_NAME) + .setImageInputStreamName(IMAGE_INPUT_STREAM) + .setOutputStreamNames(OUTPUT_STREAMS) + .setStaticImageMode(options.mode() == HandsOptions.STATIC_IMAGE_MODE) + .build(); + + initialize(context, solutionInfo, graphOutputHandler); + Map inputSidePackets = new HashMap<>(); + inputSidePackets.put(NUM_HANDS, packetCreator.createInt32(options.maxNumHands())); + start(inputSidePackets); + } + + /** + * Sets a callback to be invoked when the HandsResults become available. + * + * @param listener the {@link ResultListener} callback. + */ + public void setResultListener(ResultListener listener) { + this.graphOutputHandler.setResultListener(listener); + } + + /** + * Sets a callback to be invoked when the Hands solution throws errors. + * + * @param listener the {@link ErrorListener} callback. + */ + public void setErrorListener(@Nullable ErrorListener listener) { + this.graphOutputHandler.setErrorListener(listener); + this.errorListener = listener; + } + + /** + * Gets a specific hand landmark by hand index and hand landmark type. + * + * @param result the returned {@link HandsResult} object. + * @param handIndex the hand index. The hand landmark lists are sorted by the confidence score. + * @param landmarkType the hand landmark type defined in {@link HandLandmark}. + */ + public static NormalizedLandmark getHandLandmark( + HandsResult result, int handIndex, @HandLandmark.HandLandmarkType int landmarkType) { + if (result == null + || handIndex >= result.multiHandLandmarks().size() + || landmarkType >= HandLandmark.NUM_LANDMARKS) { + return NormalizedLandmark.getDefaultInstance(); + } + return result.multiHandLandmarks().get(handIndex).getLandmarkList().get(landmarkType); + } +} diff --git a/mediapipe/java/com/google/mediapipe/solutions/hands/HandsOptions.java b/mediapipe/java/com/google/mediapipe/solutions/hands/HandsOptions.java new file mode 100644 index 00000000..5d865723 --- /dev/null +++ b/mediapipe/java/com/google/mediapipe/solutions/hands/HandsOptions.java @@ -0,0 +1,77 @@ +// Copyright 2021 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.mediapipe.solutions.hands; + +import androidx.annotation.IntDef; +import com.google.auto.value.AutoValue; + +/** + * MediaPipe Hands solution-specific options. + * + *

mode: Whether to treat the input images as a batch of static and possibly unrelated images, or + * a video stream. See details in https://solutions.mediapipe.dev/hands#static_image_mode. + * + *

maxNumHands: Maximum number of hands to detect. See details in + * https://solutions.mediapipe.dev/hands#max_num_hands. + * + *

minDetectionConfidence: Minimum confidence value ([0.0, 1.0]) for hand detection to be + * considered successful. See details in + * https://solutions.mediapipe.dev/hands#min_detection_confidence. + * + *

minTrackingConfidence: Minimum confidence value ([0.0, 1.0]) for the hand landmarks to be + * considered tracked successfully. See details in + * https://solutions.mediapipe.dev/hands#min_tracking_confidence. + */ +@AutoValue +public abstract class HandsOptions { + + // TODO: Switch to use boolean variable. + public static final int STREAMING_MODE = 1; + public static final int STATIC_IMAGE_MODE = 2; + + /** + * Indicates whether to treat the input images as a batch of static and possibly unrelated images, + * or a video stream. + */ + @IntDef({STREAMING_MODE, STATIC_IMAGE_MODE}) + public @interface Mode {} + + @Mode + public abstract int mode(); + + public abstract int maxNumHands(); + + public abstract float minDetectionConfidence(); + + public abstract float minTrackingConfidence(); + + public static Builder builder() { + return new AutoValue_HandsOptions.Builder(); + } + + /** Builder for {@link HandsOptions}. */ + @AutoValue.Builder + public abstract static class Builder { + public abstract Builder setMode(int value); + + public abstract Builder setMaxNumHands(int value); + + public abstract Builder setMinDetectionConfidence(float value); + + public abstract Builder setMinTrackingConfidence(float value); + + public abstract HandsOptions build(); + } +} diff --git a/mediapipe/java/com/google/mediapipe/solutions/hands/HandsResult.java b/mediapipe/java/com/google/mediapipe/solutions/hands/HandsResult.java new file mode 100644 index 00000000..8b31bae8 --- /dev/null +++ b/mediapipe/java/com/google/mediapipe/solutions/hands/HandsResult.java @@ -0,0 +1,81 @@ +// Copyright 2021 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.mediapipe.solutions.hands; + +import android.graphics.Bitmap; +import com.google.auto.value.AutoBuilder; +import com.google.common.collect.ImmutableList; +import com.google.mediapipe.formats.proto.LandmarkProto.NormalizedLandmarkList; +import com.google.mediapipe.formats.proto.ClassificationProto.Classification; +import com.google.mediapipe.framework.Packet; +import com.google.mediapipe.framework.TextureFrame; +import com.google.mediapipe.solutionbase.ImageSolutionResult; +import java.util.List; + +/** + * HandsResult contains a collection of detected/tracked hands, a collection of handedness of the + * detected/tracked hands, and the input {@link Bitmap} or {@link TextureFrame}. If not in static + * image mode, the timestamp field will be set to the timestamp of the corresponding input image. + */ +public class HandsResult extends ImageSolutionResult { + private final ImmutableList multiHandLandmarks; + private final ImmutableList multiHandedness; + + HandsResult( + ImmutableList multiHandLandmarks, + ImmutableList multiHandedness, + Packet imagePacket, + long timestamp) { + this.multiHandLandmarks = multiHandLandmarks; + this.multiHandedness = multiHandedness; + this.timestamp = timestamp; + this.imagePacket = imagePacket; + } + + // Collection of detected/tracked hands, where each hand is represented as a list of 21 hand + // landmarks and each landmark is composed of x, y and z. x and y are normalized to [0.0, 1.0] by + // the image width and height respectively. z represents the landmark depth with the depth at the + // wrist being the origin, and the smaller the value the closer the landmark is to the camera. The + // magnitude of z uses roughly the same scale as x. + public ImmutableList multiHandLandmarks() { + return multiHandLandmarks; + } + + // Collection of handedness of the detected/tracked hands (i.e. is it a left or right hand). Each + // hand is composed of label and score. label is a string of value either "Left" or "Right". score + // is the estimated probability of the predicted handedness and is always greater than or equal to + // 0.5 (and the opposite handedness has an estimated probability of 1 - score). + public ImmutableList multiHandedness() { + return multiHandedness; + } + + public static Builder builder() { + return new AutoBuilder_HandsResult_Builder(); + } + + /** Builder for {@link HandsResult}. */ + @AutoBuilder + public abstract static class Builder { + abstract Builder setMultiHandLandmarks(List value); + + abstract Builder setMultiHandedness(List value); + + abstract Builder setTimestamp(long value); + + abstract Builder setImagePacket(Packet value); + + abstract HandsResult build(); + } +} diff --git a/mediapipe/modules/face_detection/BUILD b/mediapipe/modules/face_detection/BUILD index 9ee60645..839418c7 100644 --- a/mediapipe/modules/face_detection/BUILD +++ b/mediapipe/modules/face_detection/BUILD @@ -22,11 +22,11 @@ licenses(["notice"]) package(default_visibility = ["//visibility:public"]) mediapipe_simple_subgraph( - name = "face_detection_front_by_roi_cpu", - graph = "face_detection_front_by_roi_cpu.pbtxt", - register_as = "FaceDetectionFrontByRoiCpu", + name = "face_detection_short_range_by_roi_cpu", + graph = "face_detection_short_range_by_roi_cpu.pbtxt", + register_as = "FaceDetectionShortRangeByRoiCpu", deps = [ - ":face_detection_front_common", + ":face_detection_short_range_common", "//mediapipe/calculators/tensor:image_to_tensor_calculator", "//mediapipe/calculators/tensor:inference_calculator", "//mediapipe/calculators/util:to_image_calculator", @@ -34,11 +34,11 @@ mediapipe_simple_subgraph( ) mediapipe_simple_subgraph( - name = "face_detection_front_by_roi_gpu", - graph = "face_detection_front_by_roi_gpu.pbtxt", - register_as = "FaceDetectionFrontByRoiGpu", + name = "face_detection_short_range_by_roi_gpu", + graph = "face_detection_short_range_by_roi_gpu.pbtxt", + register_as = "FaceDetectionShortRangeByRoiGpu", deps = [ - ":face_detection_front_common", + ":face_detection_short_range_common", "//mediapipe/calculators/tensor:image_to_tensor_calculator", "//mediapipe/calculators/tensor:inference_calculator", "//mediapipe/calculators/util:to_image_calculator", @@ -46,11 +46,11 @@ mediapipe_simple_subgraph( ) mediapipe_simple_subgraph( - name = "face_detection_front_cpu", - graph = "face_detection_front_cpu.pbtxt", - register_as = "FaceDetectionFrontCpu", + name = "face_detection_short_range_cpu", + graph = "face_detection_short_range_cpu.pbtxt", + register_as = "FaceDetectionShortRangeCpu", deps = [ - ":face_detection_front_common", + ":face_detection_short_range_common", "//mediapipe/calculators/tensor:image_to_tensor_calculator", "//mediapipe/calculators/tensor:inference_calculator", "//mediapipe/calculators/util:to_image_calculator", @@ -58,11 +58,11 @@ mediapipe_simple_subgraph( ) mediapipe_simple_subgraph( - name = "face_detection_front_gpu", - graph = "face_detection_front_gpu.pbtxt", - register_as = "FaceDetectionFrontGpu", + name = "face_detection_short_range_gpu", + graph = "face_detection_short_range_gpu.pbtxt", + register_as = "FaceDetectionShortRangeGpu", deps = [ - ":face_detection_front_common", + ":face_detection_short_range_common", "//mediapipe/calculators/tensor:image_to_tensor_calculator", "//mediapipe/calculators/tensor:inference_calculator", "//mediapipe/calculators/util:to_image_calculator", @@ -70,9 +70,45 @@ mediapipe_simple_subgraph( ) mediapipe_simple_subgraph( - name = "face_detection_front_common", - graph = "face_detection_front_common.pbtxt", - register_as = "FaceDetectionFrontCommon", + name = "face_detection_short_range_common", + graph = "face_detection_short_range_common.pbtxt", + register_as = "FaceDetectionShortRangeCommon", + deps = [ + "//mediapipe/calculators/tensor:tensors_to_detections_calculator", + "//mediapipe/calculators/tflite:ssd_anchors_calculator", + "//mediapipe/calculators/util:detection_projection_calculator", + "//mediapipe/calculators/util:non_max_suppression_calculator", + ], +) + +mediapipe_simple_subgraph( + name = "face_detection_full_range_cpu", + graph = "face_detection_full_range_cpu.pbtxt", + register_as = "FaceDetectionFullRangeCpu", + deps = [ + ":face_detection_full_range_common", + "//mediapipe/calculators/tensor:image_to_tensor_calculator", + "//mediapipe/calculators/tensor:inference_calculator", + "//mediapipe/calculators/util:to_image_calculator", + ], +) + +mediapipe_simple_subgraph( + name = "face_detection_full_range_gpu", + graph = "face_detection_full_range_gpu.pbtxt", + register_as = "FaceDetectionFullRangeGpu", + deps = [ + ":face_detection_full_range_common", + "//mediapipe/calculators/tensor:image_to_tensor_calculator", + "//mediapipe/calculators/tensor:inference_calculator", + "//mediapipe/calculators/util:to_image_calculator", + ], +) + +mediapipe_simple_subgraph( + name = "face_detection_full_range_common", + graph = "face_detection_full_range_common.pbtxt", + register_as = "FaceDetectionFullRangeCommon", deps = [ "//mediapipe/calculators/tensor:tensors_to_detections_calculator", "//mediapipe/calculators/tflite:ssd_anchors_calculator", @@ -83,8 +119,8 @@ mediapipe_simple_subgraph( exports_files( srcs = [ - "face_detection_back.tflite", - "face_detection_back_sparse.tflite", - "face_detection_front.tflite", + "face_detection_full_range.tflite", + "face_detection_full_range_sparse.tflite", + "face_detection_short_range.tflite", ], ) diff --git a/mediapipe/modules/face_detection/README.md b/mediapipe/modules/face_detection/README.md index f4062526..17cf27b5 100644 --- a/mediapipe/modules/face_detection/README.md +++ b/mediapipe/modules/face_detection/README.md @@ -2,6 +2,7 @@ Subgraphs|Details :--- | :--- -[`FaceDetectionFrontCpu`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_detection/face_detection_front_cpu.pbtxt)| Detects faces. Works best for images from front-facing cameras (i.e. selfie images). (CPU input, and inference is executed on CPU.) -[`FaceDetectionFrontGpu`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_detection/face_detection_front_gpu.pbtxt)| Detects faces. Works best for images from front-facing cameras (i.e. selfie images). (GPU input, and inference is executed on GPU.) - +[`FaceDetectionFullRangeCpu`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_detection/face_detection_full_range_cpu.pbtxt)| Detects faces. Works best for faces within 5 meters from the camera. (CPU input, and inference is executed on CPU.) +[`FaceDetectionFullRangeGpu`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_detection/face_detection_full_range_gpu.pbtxt)| Detects faces. Works best for faces within 5 meters from the camera. (GPU input, and inference is executed on GPU.) +[`FaceDetectionShortRangeCpu`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_detection/face_detection_short_range_cpu.pbtxt)| Detects faces. Works best for faces within 2 meters from the camera. (CPU input, and inference is executed on CPU.) +[`FaceDetectionShortRangeGpu`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_detection/face_detection_short_range_gpu.pbtxt)| Detects faces. Works best for faces within 2 meters from the camera. (GPU input, and inference is executed on GPU.) diff --git a/mediapipe/modules/face_detection/face_detection_back.tflite b/mediapipe/modules/face_detection/face_detection_full_range.tflite similarity index 100% rename from mediapipe/modules/face_detection/face_detection_back.tflite rename to mediapipe/modules/face_detection/face_detection_full_range.tflite diff --git a/mediapipe/modules/face_detection/face_detection_full_range_common.pbtxt b/mediapipe/modules/face_detection/face_detection_full_range_common.pbtxt new file mode 100644 index 00000000..937e8be1 --- /dev/null +++ b/mediapipe/modules/face_detection/face_detection_full_range_common.pbtxt @@ -0,0 +1,102 @@ +# MediaPipe graph performing common processing to detect faces using +# face_detection_full_range_sparse.tflite model, currently consisting of tensor +# post processing. +# +# EXAMPLE: +# node { +# calculator: "FaceDetectionFullRangeCommon" +# input_stream: "TENSORS:detection_tensors" +# input_stream: "MATRIX:transform_matrix" +# output_stream: "DETECTIONS:detections" +# } + +type: "FaceDetectionShortRangeCommon" + +# Detection tensors. (std::vector) +input_stream: "TENSORS:detection_tensors" + +# A 4x4 row-major-order matrix that maps a point represented in the detection +# tensors to a desired coordinate system, e.g., in the original input image +# before scaling/cropping. (std::array) +input_stream: "MATRIX:transform_matrix" + +# Detected faces. (std::vector) +# NOTE: there will not be an output packet in the DETECTIONS stream for this +# particular timestamp if none of faces detected. However, the MediaPipe +# framework will internally inform the downstream calculators of the absence of +# this packet so that they don't wait for it unnecessarily. +output_stream: "DETECTIONS:detections" + +# Generates a single side packet containing a vector of SSD anchors based on +# the specification in the options. +node { + calculator: "SsdAnchorsCalculator" + output_side_packet: "anchors" + options: { + [mediapipe.SsdAnchorsCalculatorOptions.ext] { + num_layers: 1 + min_scale: 0.1484375 + max_scale: 0.75 + input_size_height: 192 + input_size_width: 192 + anchor_offset_x: 0.5 + anchor_offset_y: 0.5 + strides: 4 + aspect_ratios: 1.0 + fixed_anchor_size: true + interpolated_scale_aspect_ratio: 0.0 + } + } +} + +# Decodes the detection tensors generated by the TensorFlow Lite model, based on +# the SSD anchors and the specification in the options, into a vector of +# detections. Each detection describes a detected object. +node { + calculator: "TensorsToDetectionsCalculator" + input_stream: "TENSORS:detection_tensors" + input_side_packet: "ANCHORS:anchors" + output_stream: "DETECTIONS:unfiltered_detections" + options: { + [mediapipe.TensorsToDetectionsCalculatorOptions.ext] { + num_classes: 1 + num_boxes: 2304 + num_coords: 16 + box_coord_offset: 0 + keypoint_coord_offset: 4 + num_keypoints: 6 + num_values_per_keypoint: 2 + sigmoid_score: true + score_clipping_thresh: 100.0 + reverse_output_order: true + x_scale: 192.0 + y_scale: 192.0 + h_scale: 192.0 + w_scale: 192.0 + min_score_thresh: 0.6 + } + } +} + +# Performs non-max suppression to remove excessive detections. +node { + calculator: "NonMaxSuppressionCalculator" + input_stream: "unfiltered_detections" + output_stream: "filtered_detections" + options: { + [mediapipe.NonMaxSuppressionCalculatorOptions.ext] { + min_suppression_threshold: 0.3 + overlap_type: INTERSECTION_OVER_UNION + algorithm: WEIGHTED + } + } +} + +# Projects the detections from input tensor to the corresponding locations on +# the original image (input to the graph). +node { + calculator: "DetectionProjectionCalculator" + input_stream: "DETECTIONS:filtered_detections" + input_stream: "PROJECTION_MATRIX:transform_matrix" + output_stream: "DETECTIONS:detections" +} diff --git a/mediapipe/modules/face_detection/face_detection_full_range_cpu.pbtxt b/mediapipe/modules/face_detection/face_detection_full_range_cpu.pbtxt new file mode 100644 index 00000000..13d47402 --- /dev/null +++ b/mediapipe/modules/face_detection/face_detection_full_range_cpu.pbtxt @@ -0,0 +1,79 @@ +# MediaPipe graph to detect faces. (CPU input, and inference is executed on +# CPU.) +# +# It is required that "face_detection_full_range_sparse.tflite" is available at +# "mediapipe/modules/face_detection/face_detection_full_range_sparse.tflite" +# path during execution. +# +# EXAMPLE: +# node { +# calculator: "FaceDetectionFullRangeCpu" +# input_stream: "IMAGE:image" +# output_stream: "DETECTIONS:face_detections" +# } + +type: "FaceDetectionFullRangeCpu" + +# CPU image. (ImageFrame) +input_stream: "IMAGE:image" + +# Detected faces. (std::vector) +# NOTE: there will not be an output packet in the DETECTIONS stream for this +# particular timestamp if none of faces detected. However, the MediaPipe +# framework will internally inform the downstream calculators of the absence of +# this packet so that they don't wait for it unnecessarily. +output_stream: "DETECTIONS:detections" + +# Converts the input CPU image (ImageFrame) to the multi-backend image type +# (Image). +node: { + calculator: "ToImageCalculator" + input_stream: "IMAGE_CPU:image" + output_stream: "IMAGE:multi_backend_image" +} + +# Transforms the input image into a 192x192 tensor while keeping the aspect +# ratio (what is expected by the corresponding face detection model), resulting +# in potential letterboxing in the transformed image. +node: { + calculator: "ImageToTensorCalculator" + input_stream: "IMAGE:multi_backend_image" + output_stream: "TENSORS:input_tensors" + output_stream: "MATRIX:transform_matrix" + options: { + [mediapipe.ImageToTensorCalculatorOptions.ext] { + output_tensor_width: 192 + output_tensor_height: 192 + keep_aspect_ratio: true + output_tensor_float_range { + min: -1.0 + max: 1.0 + } + border_mode: BORDER_ZERO + } + } +} + +# Runs a TensorFlow Lite model on CPU that takes an image tensor and outputs a +# vector of tensors representing, for instance, detection boxes/keypoints and +# scores. +node { + calculator: "InferenceCalculator" + input_stream: "TENSORS:input_tensors" + output_stream: "TENSORS:detection_tensors" + options: { + [mediapipe.InferenceCalculatorOptions.ext] { + model_path: "mediapipe/modules/face_detection/face_detection_full_range_sparse.tflite" + delegate { xnnpack {} } + } + # + } +} + +# Performs tensor post processing to generate face detections. +node { + calculator: "FaceDetectionFullRangeCommon" + input_stream: "TENSORS:detection_tensors" + input_stream: "MATRIX:transform_matrix" + output_stream: "DETECTIONS:detections" +} diff --git a/mediapipe/modules/face_detection/face_detection_full_range_gpu.pbtxt b/mediapipe/modules/face_detection/face_detection_full_range_gpu.pbtxt new file mode 100644 index 00000000..703b7178 --- /dev/null +++ b/mediapipe/modules/face_detection/face_detection_full_range_gpu.pbtxt @@ -0,0 +1,80 @@ +# MediaPipe graph to detect faces. (GPU input, and inference is executed on +# GPU.) +# +# It is required that "face_detection_full_range_sparse.tflite" is available at +# "mediapipe/modules/face_detection/face_detection_full_range_sparse.tflite" +# path during execution. +# +# EXAMPLE: +# node { +# calculator: "FaceDetectionFullRangeGpu" +# input_stream: "IMAGE:image" +# output_stream: "DETECTIONS:face_detections" +# } + +type: "FaceDetectionFullRangeGpu" + +# GPU image. (GpuBuffer) +input_stream: "IMAGE:image" + +# Detected faces. (std::vector) +# NOTE: there will not be an output packet in the DETECTIONS stream for this +# particular timestamp if none of faces detected. However, the MediaPipe +# framework will internally inform the downstream calculators of the absence of +# this packet so that they don't wait for it unnecessarily. +output_stream: "DETECTIONS:detections" + +# Converts the input GPU image (GpuBuffer) to the multi-backend image type +# (Image). +node: { + calculator: "ToImageCalculator" + input_stream: "IMAGE_GPU:image" + output_stream: "IMAGE:multi_backend_image" +} + +# Transforms the input image into a 128x128 tensor while keeping the aspect +# ratio (what is expected by the corresponding face detection model), resulting +# in potential letterboxing in the transformed image. +node: { + calculator: "ImageToTensorCalculator" + input_stream: "IMAGE:multi_backend_image" + output_stream: "TENSORS:input_tensors" + output_stream: "MATRIX:transform_matrix" + options: { + [mediapipe.ImageToTensorCalculatorOptions.ext] { + output_tensor_width: 192 + output_tensor_height: 192 + keep_aspect_ratio: true + output_tensor_float_range { + min: -1.0 + max: 1.0 + } + border_mode: BORDER_ZERO + gpu_origin: TOP_LEFT + } + } +} + +# Runs a TensorFlow Lite model on GPU that takes an image tensor and outputs a +# vector of tensors representing, for instance, detection boxes/keypoints and +# scores. +node { + calculator: "InferenceCalculator" + input_stream: "TENSORS:input_tensors" + output_stream: "TENSORS:detection_tensors" + options: { + [mediapipe.InferenceCalculatorOptions.ext] { + model_path: "mediapipe/modules/face_detection/face_detection_full_range_sparse.tflite" + # + delegate: { gpu { use_advanced_gpu_api: true } } + } + } +} + +# Performs tensor post processing to generate face detections. +node { + calculator: "FaceDetectionFullRangeCommon" + input_stream: "TENSORS:detection_tensors" + input_stream: "MATRIX:transform_matrix" + output_stream: "DETECTIONS:detections" +} diff --git a/mediapipe/modules/face_detection/face_detection_back_sparse.tflite b/mediapipe/modules/face_detection/face_detection_full_range_sparse.tflite similarity index 100% rename from mediapipe/modules/face_detection/face_detection_back_sparse.tflite rename to mediapipe/modules/face_detection/face_detection_full_range_sparse.tflite diff --git a/mediapipe/modules/face_detection/face_detection_front.tflite b/mediapipe/modules/face_detection/face_detection_short_range.tflite similarity index 100% rename from mediapipe/modules/face_detection/face_detection_front.tflite rename to mediapipe/modules/face_detection/face_detection_short_range.tflite diff --git a/mediapipe/modules/face_detection/face_detection_front_by_roi_cpu.pbtxt b/mediapipe/modules/face_detection/face_detection_short_range_by_roi_cpu.pbtxt similarity index 88% rename from mediapipe/modules/face_detection/face_detection_front_by_roi_cpu.pbtxt rename to mediapipe/modules/face_detection/face_detection_short_range_by_roi_cpu.pbtxt index 06f652f8..b3adfeb8 100644 --- a/mediapipe/modules/face_detection/face_detection_front_by_roi_cpu.pbtxt +++ b/mediapipe/modules/face_detection/face_detection_short_range_by_roi_cpu.pbtxt @@ -1,19 +1,19 @@ # MediaPipe graph to detect faces. (CPU input, and inference is executed on # CPU.) # -# It is required that "face_detection_front.tflite" is available at -# "mediapipe/modules/face_detection/face_detection_front.tflite" +# It is required that "face_detection_short_range.tflite" is available at +# "mediapipe/modules/face_detection/face_detection_short_range.tflite" # path during execution. # # EXAMPLE: # node { -# calculator: "FaceDetectionFrontByRoiCpu" +# calculator: "FaceDetectionShortRangeByRoiCpu" # input_stream: "IMAGE:image" # input_stream: "ROI:roi" # output_stream: "DETECTIONS:face_detections" # } -type: "FaceDetectionFrontByRoiCpu" +type: "FaceDetectionShortRangeByRoiCpu" # CPU image. (ImageFrame) input_stream: "IMAGE:image" @@ -68,7 +68,7 @@ node { output_stream: "TENSORS:detection_tensors" options: { [mediapipe.InferenceCalculatorOptions.ext] { - model_path: "mediapipe/modules/face_detection/face_detection_front.tflite" + model_path: "mediapipe/modules/face_detection/face_detection_short_range.tflite" delegate { xnnpack {} } } } @@ -76,7 +76,7 @@ node { # Performs tensor post processing to generate face detections. node { - calculator: "FaceDetectionFrontCommon" + calculator: "FaceDetectionShortRangeCommon" input_stream: "TENSORS:detection_tensors" input_stream: "MATRIX:transform_matrix" output_stream: "DETECTIONS:detections" diff --git a/mediapipe/modules/face_detection/face_detection_front_by_roi_gpu.pbtxt b/mediapipe/modules/face_detection/face_detection_short_range_by_roi_gpu.pbtxt similarity index 88% rename from mediapipe/modules/face_detection/face_detection_front_by_roi_gpu.pbtxt rename to mediapipe/modules/face_detection/face_detection_short_range_by_roi_gpu.pbtxt index 179ace81..1bd08e93 100644 --- a/mediapipe/modules/face_detection/face_detection_front_by_roi_gpu.pbtxt +++ b/mediapipe/modules/face_detection/face_detection_short_range_by_roi_gpu.pbtxt @@ -1,19 +1,19 @@ # MediaPipe graph to detect faces. (CPU input, and inference is executed on # CPU.) # -# It is required that "face_detection_front.tflite" is available at -# "mediapipe/modules/face_detection/face_detection_front.tflite" +# It is required that "face_detection_short_range.tflite" is available at +# "mediapipe/modules/face_detection/face_detection_short_range.tflite" # path during execution. # # EXAMPLE: # node { -# calculator: "FaceDetectionFrontByRoiGpu" +# calculator: "FaceDetectionShortRangeByRoiGpu" # input_stream: "IMAGE:image" # input_stream: "ROI:roi" # output_stream: "DETECTIONS:face_detections" # } -type: "FaceDetectionFrontByRoiGpu" +type: "FaceDetectionShortRangeByRoiGpu" # GPU image. (GpuBuffer) input_stream: "IMAGE:image" @@ -69,14 +69,14 @@ node { output_stream: "TENSORS:detection_tensors" options: { [mediapipe.InferenceCalculatorOptions.ext] { - model_path: "mediapipe/modules/face_detection/face_detection_front.tflite" + model_path: "mediapipe/modules/face_detection/face_detection_short_range.tflite" } } } # Performs tensor post processing to generate face detections. node { - calculator: "FaceDetectionFrontCommon" + calculator: "FaceDetectionShortRangeCommon" input_stream: "TENSORS:detection_tensors" input_stream: "MATRIX:transform_matrix" output_stream: "DETECTIONS:detections" diff --git a/mediapipe/modules/face_detection/face_detection_front_common.pbtxt b/mediapipe/modules/face_detection/face_detection_short_range_common.pbtxt similarity index 97% rename from mediapipe/modules/face_detection/face_detection_front_common.pbtxt rename to mediapipe/modules/face_detection/face_detection_short_range_common.pbtxt index 3c920059..4a6a54f4 100644 --- a/mediapipe/modules/face_detection/face_detection_front_common.pbtxt +++ b/mediapipe/modules/face_detection/face_detection_short_range_common.pbtxt @@ -3,13 +3,13 @@ # # EXAMPLE: # node { -# calculator: "FaceDetectionFrontCommon" +# calculator: "FaceDetectionShortRangeCommon" # input_stream: "TENSORS:detection_tensors" # input_stream: "MATRIX:transform_matrix" # output_stream: "DETECTIONS:detections" # } -type: "FaceDetectionFrontCommon" +type: "FaceDetectionShortRangeCommon" # Detection tensors. (std::vector) input_stream: "TENSORS:detection_tensors" diff --git a/mediapipe/modules/face_detection/face_detection_front_cpu.pbtxt b/mediapipe/modules/face_detection/face_detection_short_range_cpu.pbtxt similarity index 88% rename from mediapipe/modules/face_detection/face_detection_front_cpu.pbtxt rename to mediapipe/modules/face_detection/face_detection_short_range_cpu.pbtxt index bfae6162..0db24204 100644 --- a/mediapipe/modules/face_detection/face_detection_front_cpu.pbtxt +++ b/mediapipe/modules/face_detection/face_detection_short_range_cpu.pbtxt @@ -1,18 +1,18 @@ # MediaPipe graph to detect faces. (CPU input, and inference is executed on # CPU.) # -# It is required that "face_detection_front.tflite" is available at -# "mediapipe/modules/face_detection/face_detection_front.tflite" +# It is required that "face_detection_short_range.tflite" is available at +# "mediapipe/modules/face_detection/face_detection_short_range.tflite" # path during execution. # # EXAMPLE: # node { -# calculator: "FaceDetectionFrontCpu" +# calculator: "FaceDetectionShortRangeCpu" # input_stream: "IMAGE:image" # output_stream: "DETECTIONS:face_detections" # } -type: "FaceDetectionFrontCpu" +type: "FaceDetectionShortRangeCpu" # CPU image. (ImageFrame) input_stream: "IMAGE:image" @@ -63,7 +63,7 @@ node { output_stream: "TENSORS:detection_tensors" options: { [mediapipe.InferenceCalculatorOptions.ext] { - model_path: "mediapipe/modules/face_detection/face_detection_front.tflite" + model_path: "mediapipe/modules/face_detection/face_detection_short_range.tflite" delegate { xnnpack {} } } } @@ -71,7 +71,7 @@ node { # Performs tensor post processing to generate face detections. node { - calculator: "FaceDetectionFrontCommon" + calculator: "FaceDetectionShortRangeCommon" input_stream: "TENSORS:detection_tensors" input_stream: "MATRIX:transform_matrix" output_stream: "DETECTIONS:detections" diff --git a/mediapipe/modules/face_detection/face_detection_front_gpu.pbtxt b/mediapipe/modules/face_detection/face_detection_short_range_gpu.pbtxt similarity index 88% rename from mediapipe/modules/face_detection/face_detection_front_gpu.pbtxt rename to mediapipe/modules/face_detection/face_detection_short_range_gpu.pbtxt index 21a9158b..d30644b1 100644 --- a/mediapipe/modules/face_detection/face_detection_front_gpu.pbtxt +++ b/mediapipe/modules/face_detection/face_detection_short_range_gpu.pbtxt @@ -1,18 +1,18 @@ # MediaPipe graph to detect faces. (CPU input, and inference is executed on # CPU.) # -# It is required that "face_detection_front.tflite" is available at -# "mediapipe/modules/face_detection/face_detection_front.tflite" +# It is required that "face_detection_short_range.tflite" is available at +# "mediapipe/modules/face_detection/face_detection_short_range.tflite" # path during execution. # # EXAMPLE: # node { -# calculator: "FaceDetectionFrontGpu" +# calculator: "FaceDetectionShortRangeGpu" # input_stream: "IMAGE:image" # output_stream: "DETECTIONS:face_detections" # } -type: "FaceDetectionFrontGpu" +type: "FaceDetectionShortRangeGpu" # GPU image. (GpuBuffer) input_stream: "IMAGE:image" @@ -64,14 +64,14 @@ node { output_stream: "TENSORS:detection_tensors" options: { [mediapipe.InferenceCalculatorOptions.ext] { - model_path: "mediapipe/modules/face_detection/face_detection_front.tflite" + model_path: "mediapipe/modules/face_detection/face_detection_short_range.tflite" } } } # Performs tensor post processing to generate face detections. node { - calculator: "FaceDetectionFrontCommon" + calculator: "FaceDetectionShortRangeCommon" input_stream: "TENSORS:detection_tensors" input_stream: "MATRIX:transform_matrix" output_stream: "DETECTIONS:detections" diff --git a/mediapipe/modules/face_landmark/BUILD b/mediapipe/modules/face_landmark/BUILD index 7aa10fe5..77560022 100644 --- a/mediapipe/modules/face_landmark/BUILD +++ b/mediapipe/modules/face_landmark/BUILD @@ -70,7 +70,7 @@ mediapipe_simple_subgraph( "//mediapipe/calculators/image:image_properties_calculator", "//mediapipe/calculators/util:association_norm_rect_calculator", "//mediapipe/calculators/util:collection_has_min_size_calculator", - "//mediapipe/modules/face_detection:face_detection_front_cpu", + "//mediapipe/modules/face_detection:face_detection_short_range_cpu", ], ) @@ -91,7 +91,7 @@ mediapipe_simple_subgraph( "//mediapipe/calculators/image:image_properties_calculator", "//mediapipe/calculators/util:association_norm_rect_calculator", "//mediapipe/calculators/util:collection_has_min_size_calculator", - "//mediapipe/modules/face_detection:face_detection_front_gpu", + "//mediapipe/modules/face_detection:face_detection_short_range_gpu", ], ) diff --git a/mediapipe/modules/face_landmark/face_detection_front_detection_to_roi.pbtxt b/mediapipe/modules/face_landmark/face_detection_front_detection_to_roi.pbtxt index 133bc9a9..acc94765 100644 --- a/mediapipe/modules/face_landmark/face_detection_front_detection_to_roi.pbtxt +++ b/mediapipe/modules/face_landmark/face_detection_front_detection_to_roi.pbtxt @@ -1,6 +1,6 @@ # MediaPipe graph to calculate face region of interest (ROI) from the very # first face detection in the vector of detections provided by -# "FaceDetectionFrontCpu" or "FaceDetectionFrontGpu" +# "FaceDetectionShortRangeCpu" or "FaceDetectionShortRangeGpu" # # NOTE: this graph is subject to change and should not be used directly. diff --git a/mediapipe/modules/face_landmark/face_landmark_front_cpu.pbtxt b/mediapipe/modules/face_landmark/face_landmark_front_cpu.pbtxt index 6086a7cd..f60ca3df 100644 --- a/mediapipe/modules/face_landmark/face_landmark_front_cpu.pbtxt +++ b/mediapipe/modules/face_landmark/face_landmark_front_cpu.pbtxt @@ -2,8 +2,8 @@ # executed on CPU.) This graph tries to skip face detection as much as possible # by using previously detected/predicted landmarks for new images. # -# It is required that "face_detection_front.tflite" is available at -# "mediapipe/modules/face_detection/face_detection_front.tflite" +# It is required that "face_detection_short_range.tflite" is available at +# "mediapipe/modules/face_detection/face_detection_short_range.tflite" # path during execution. # # It is required that "face_landmark.tflite" is available at @@ -89,7 +89,7 @@ node { # Detects faces. node { - calculator: "FaceDetectionFrontCpu" + calculator: "FaceDetectionShortRangeCpu" input_stream: "IMAGE:gated_image" output_stream: "DETECTIONS:all_face_detections" } diff --git a/mediapipe/modules/face_landmark/face_landmark_front_gpu.pbtxt b/mediapipe/modules/face_landmark/face_landmark_front_gpu.pbtxt index 1caa6baf..fe93d195 100644 --- a/mediapipe/modules/face_landmark/face_landmark_front_gpu.pbtxt +++ b/mediapipe/modules/face_landmark/face_landmark_front_gpu.pbtxt @@ -2,8 +2,8 @@ # executed on GPU.) This graph tries to skip face detection as much as possible # by using previously detected/predicted landmarks for new images. # -# It is required that "face_detection_front.tflite" is available at -# "mediapipe/modules/face_detection/face_detection_front.tflite" +# It is required that "face_detection_short_range.tflite" is available at +# "mediapipe/modules/face_detection/face_detection_short_range.tflite" # path during execution. # # It is required that "face_landmark.tflite" is available at @@ -89,7 +89,7 @@ node { # Detects faces. node { - calculator: "FaceDetectionFrontGpu" + calculator: "FaceDetectionShortRangeGpu" input_stream: "IMAGE:gated_image" output_stream: "DETECTIONS:all_face_detections" } diff --git a/mediapipe/modules/face_landmark/face_landmark_front_side_model_cpu.pbtxt b/mediapipe/modules/face_landmark/face_landmark_front_side_model_cpu.pbtxt new file mode 100644 index 00000000..d3d26c04 --- /dev/null +++ b/mediapipe/modules/face_landmark/face_landmark_front_side_model_cpu.pbtxt @@ -0,0 +1,224 @@ +# MediaPipe graph to detect/predict face landmarks. (CPU input, and inference is +# executed on CPU.) This graph tries to skip face detection as much as possible +# by using previously detected/predicted landmarks for new images. +# +# EXAMPLE: +# node { +# calculator: "FaceLandmarkFrontSideModelCpu" +# input_stream: "IMAGE:image" +# input_side_packet: "NUM_FACES:num_faces" +# input_side_packet: "MODEL:0:face_detection_model" +# input_side_packet: "MODEL:1:face_landmark_model" +# output_stream: "LANDMARKS:multi_face_landmarks" +# } + +type: "FaceLandmarkFrontSideModelCpu" + +# CPU image. (ImageFrame) +input_stream: "IMAGE:image" + +# Max number of faces to detect/track. (int) +input_side_packet: "NUM_FACES:num_faces" +# TfLite model to detect faces. +# (std::unique_ptr>) +# NOTE: mediapipe/modules/face_detection/face_detection_short_range.tflite +# model only, can be passed here, otherwise - results are undefined. +input_side_packet: "MODEL:0:face_detection_model" +# TfLite model to detect face landmarks. +# (std::unique_ptr>) +# NOTE: mediapipe/modules/face_landmark/face_landmark.tflite model +# only, can be passed here, otherwise - results are undefined. +input_side_packet: "MODEL:1:face_landmark_model" + +# Collection of detected/predicted faces, each represented as a list of 468 face +# landmarks. (std::vector) +# NOTE: there will not be an output packet in the LANDMARKS stream for this +# particular timestamp if none of faces detected. However, the MediaPipe +# framework will internally inform the downstream calculators of the absence of +# this packet so that they don't wait for it unnecessarily. +output_stream: "LANDMARKS:multi_face_landmarks" + +# Extra outputs (for debugging, for instance). +# Detected faces. (std::vector) +output_stream: "DETECTIONS:face_detections" +# Regions of interest calculated based on landmarks. +# (std::vector) +output_stream: "ROIS_FROM_LANDMARKS:face_rects_from_landmarks" +# Regions of interest calculated based on face detections. +# (std::vector) +output_stream: "ROIS_FROM_DETECTIONS:face_rects_from_detections" + +# Determines if an input vector of NormalizedRect has a size greater than or +# equal to the provided num_faces. +node { + calculator: "NormalizedRectVectorHasMinSizeCalculator" + input_stream: "ITERABLE:prev_face_rects_from_landmarks" + input_side_packet: "num_faces" + output_stream: "prev_has_enough_faces" +} + +# Drops the incoming image if FaceLandmarkCpu was able to identify face presence +# in the previous image. Otherwise, passes the incoming image through to trigger +# a new round of face detection in FaceDetectionShortRangeCpu. +node { + calculator: "GateCalculator" + input_stream: "image" + input_stream: "DISALLOW:prev_has_enough_faces" + output_stream: "gated_image" + options: { + [mediapipe.GateCalculatorOptions.ext] { + empty_packets_as_allow: true + } + } +} + +# Detects faces. +node { + calculator: "FaceDetectionShortRangeSideModelCpu" + input_stream: "IMAGE:gated_image" + input_side_packet: "MODEL:face_detection_model" + output_stream: "DETECTIONS:all_face_detections" +} + +# Makes sure there are no more detections than the provided num_faces. +node { + calculator: "ClipDetectionVectorSizeCalculator" + input_stream: "all_face_detections" + output_stream: "face_detections" + input_side_packet: "num_faces" +} + +# Calculate size of the image. +node { + calculator: "ImagePropertiesCalculator" + input_stream: "IMAGE:gated_image" + output_stream: "SIZE:gated_image_size" +} + +# Outputs each element of face_detections at a fake timestamp for the rest of +# the graph to process. Clones the image size packet for each face_detection at +# the fake timestamp. At the end of the loop, outputs the BATCH_END timestamp +# for downstream calculators to inform them that all elements in the vector have +# been processed. +node { + calculator: "BeginLoopDetectionCalculator" + input_stream: "ITERABLE:face_detections" + input_stream: "CLONE:gated_image_size" + output_stream: "ITEM:face_detection" + output_stream: "CLONE:detections_loop_image_size" + output_stream: "BATCH_END:detections_loop_end_timestamp" +} + +# Calculates region of interest based on face detections, so that can be used +# to detect landmarks. +node { + calculator: "FaceDetectionFrontDetectionToRoi" + input_stream: "DETECTION:face_detection" + input_stream: "IMAGE_SIZE:detections_loop_image_size" + output_stream: "ROI:face_rect_from_detection" +} + +# Collects a NormalizedRect for each face into a vector. Upon receiving the +# BATCH_END timestamp, outputs the vector of NormalizedRect at the BATCH_END +# timestamp. +node { + calculator: "EndLoopNormalizedRectCalculator" + input_stream: "ITEM:face_rect_from_detection" + input_stream: "BATCH_END:detections_loop_end_timestamp" + output_stream: "ITERABLE:face_rects_from_detections" +} + +# Performs association between NormalizedRect vector elements from previous +# image and rects based on face detections from the current image. This +# calculator ensures that the output face_rects vector doesn't contain +# overlapping regions based on the specified min_similarity_threshold. +node { + calculator: "AssociationNormRectCalculator" + input_stream: "face_rects_from_detections" + input_stream: "prev_face_rects_from_landmarks" + output_stream: "face_rects" + options: { + [mediapipe.AssociationCalculatorOptions.ext] { + min_similarity_threshold: 0.5 + } + } +} + +# Calculate size of the image. +node { + calculator: "ImagePropertiesCalculator" + input_stream: "IMAGE:image" + output_stream: "SIZE:image_size" +} + +# Outputs each element of face_rects at a fake timestamp for the rest of the +# graph to process. Clones image and image size packets for each +# single_face_rect at the fake timestamp. At the end of the loop, outputs the +# BATCH_END timestamp for downstream calculators to inform them that all +# elements in the vector have been processed. +node { + calculator: "BeginLoopNormalizedRectCalculator" + input_stream: "ITERABLE:face_rects" + input_stream: "CLONE:0:image" + input_stream: "CLONE:1:image_size" + output_stream: "ITEM:face_rect" + output_stream: "CLONE:0:landmarks_loop_image" + output_stream: "CLONE:1:landmarks_loop_image_size" + output_stream: "BATCH_END:landmarks_loop_end_timestamp" +} + +# Detects face landmarks within specified region of interest of the image. +node { + calculator: "FaceLandmarkSideModelCpu" + input_stream: "IMAGE:landmarks_loop_image" + input_stream: "ROI:face_rect" + input_side_packet: "MODEL:face_landmark_model" + output_stream: "LANDMARKS:face_landmarks" +} + +# Calculates region of interest based on face landmarks, so that can be reused +# for subsequent image. +node { + calculator: "FaceLandmarkLandmarksToRoi" + input_stream: "LANDMARKS:face_landmarks" + input_stream: "IMAGE_SIZE:landmarks_loop_image_size" + output_stream: "ROI:face_rect_from_landmarks" +} + +# Collects a set of landmarks for each face into a vector. Upon receiving the +# BATCH_END timestamp, outputs the vector of landmarks at the BATCH_END +# timestamp. +node { + calculator: "EndLoopNormalizedLandmarkListVectorCalculator" + input_stream: "ITEM:face_landmarks" + input_stream: "BATCH_END:landmarks_loop_end_timestamp" + output_stream: "ITERABLE:multi_face_landmarks" +} + +# Collects a NormalizedRect for each face into a vector. Upon receiving the +# BATCH_END timestamp, outputs the vector of NormalizedRect at the BATCH_END +# timestamp. +node { + calculator: "EndLoopNormalizedRectCalculator" + input_stream: "ITEM:face_rect_from_landmarks" + input_stream: "BATCH_END:landmarks_loop_end_timestamp" + output_stream: "ITERABLE:face_rects_from_landmarks" +} + +# Caches face rects calculated from landmarks, and upon the arrival of the next +# input image, sends out the cached rects with timestamps replaced by that of +# the input image, essentially generating a packet that carries the previous +# face rects. Note that upon the arrival of the very first input image, a +# timestamp bound update occurs to jump start the feedback loop. +node { + calculator: "PreviousLoopbackCalculator" + input_stream: "MAIN:image" + input_stream: "LOOP:face_rects_from_landmarks" + input_stream_info: { + tag_index: "LOOP" + back_edge: true + } + output_stream: "PREV_LOOP:prev_face_rects_from_landmarks" +} diff --git a/mediapipe/modules/face_landmark/face_landmark_front_side_model_gpu.pbtxt b/mediapipe/modules/face_landmark/face_landmark_front_side_model_gpu.pbtxt new file mode 100644 index 00000000..9832c2f0 --- /dev/null +++ b/mediapipe/modules/face_landmark/face_landmark_front_side_model_gpu.pbtxt @@ -0,0 +1,224 @@ +# MediaPipe graph to detect/predict face landmarks. (GPU input, and inference is +# executed on GPU.) This graph tries to skip face detection as much as possible +# by using previously detected/predicted landmarks for new images. +# +# EXAMPLE: +# node { +# calculator: "FaceLandmarkFrontSideModelGpu" +# input_stream: "IMAGE:image" +# input_side_packet: "NUM_FACES:num_faces" +# input_side_packet: "MODEL:0:face_detection_model" +# input_side_packet: "MODEL:1:face_landmark_model" +# output_stream: "LANDMARKS:multi_face_landmarks" +# } + +type: "FaceLandmarkFrontSideModelGpu" + +# GPU image. (GpuBuffer) +input_stream: "IMAGE:image" + +# Max number of faces to detect/track. (int) +input_side_packet: "NUM_FACES:num_faces" +# TfLite model to detect faces. +# (std::unique_ptr>) +# NOTE: mediapipe/modules/face_detection/face_detection_short_range.tflite +# model only, can be passed here, otherwise - results are undefined. +input_side_packet: "MODEL:0:face_detection_model" +# TfLite model to detect face landmarks. +# (std::unique_ptr>) +# NOTE: mediapipe/modules/face_landmark/face_landmark.tflite model +# only, can be passed here, otherwise - results are undefined. +input_side_packet: "MODEL:1:face_landmark_model" + +# Collection of detected/predicted faces, each represented as a list of 468 face +# landmarks. (std::vector) +# NOTE: there will not be an output packet in the LANDMARKS stream for this +# particular timestamp if none of faces detected. However, the MediaPipe +# framework will internally inform the downstream calculators of the absence of +# this packet so that they don't wait for it unnecessarily. +output_stream: "LANDMARKS:multi_face_landmarks" + +# Extra outputs (for debugging, for instance). +# Detected faces. (std::vector) +output_stream: "DETECTIONS:face_detections" +# Regions of interest calculated based on landmarks. +# (std::vector) +output_stream: "ROIS_FROM_LANDMARKS:face_rects_from_landmarks" +# Regions of interest calculated based on face detections. +# (std::vector) +output_stream: "ROIS_FROM_DETECTIONS:face_rects_from_detections" + +# Determines if an input vector of NormalizedRect has a size greater than or +# equal to the provided num_faces. +node { + calculator: "NormalizedRectVectorHasMinSizeCalculator" + input_stream: "ITERABLE:prev_face_rects_from_landmarks" + input_side_packet: "num_faces" + output_stream: "prev_has_enough_faces" +} + +# Drops the incoming image if FaceLandmarkGpu was able to identify face presence +# in the previous image. Otherwise, passes the incoming image through to trigger +# a new round of face detection in FaceDetectionShortRangeGpu. +node { + calculator: "GateCalculator" + input_stream: "image" + input_stream: "DISALLOW:prev_has_enough_faces" + output_stream: "gated_image" + options: { + [mediapipe.GateCalculatorOptions.ext] { + empty_packets_as_allow: true + } + } +} + +# Detects faces. +node { + calculator: "FaceDetectionShortRangeSideModelGpu" + input_stream: "IMAGE:gated_image" + input_side_packet: "MODEL:face_detection_model" + output_stream: "DETECTIONS:all_face_detections" +} + +# Makes sure there are no more detections than the provided num_faces. +node { + calculator: "ClipDetectionVectorSizeCalculator" + input_stream: "all_face_detections" + output_stream: "face_detections" + input_side_packet: "num_faces" +} + +# Calculate size of the image. +node { + calculator: "ImagePropertiesCalculator" + input_stream: "IMAGE_GPU:gated_image" + output_stream: "SIZE:gated_image_size" +} + +# Outputs each element of face_detections at a fake timestamp for the rest of +# the graph to process. Clones the image size packet for each face_detection at +# the fake timestamp. At the end of the loop, outputs the BATCH_END timestamp +# for downstream calculators to inform them that all elements in the vector have +# been processed. +node { + calculator: "BeginLoopDetectionCalculator" + input_stream: "ITERABLE:face_detections" + input_stream: "CLONE:gated_image_size" + output_stream: "ITEM:face_detection" + output_stream: "CLONE:detections_loop_image_size" + output_stream: "BATCH_END:detections_loop_end_timestamp" +} + +# Calculates region of interest based on face detections, so that can be used +# to detect landmarks. +node { + calculator: "FaceDetectionFrontDetectionToRoi" + input_stream: "DETECTION:face_detection" + input_stream: "IMAGE_SIZE:detections_loop_image_size" + output_stream: "ROI:face_rect_from_detection" +} + +# Collects a NormalizedRect for each face into a vector. Upon receiving the +# BATCH_END timestamp, outputs the vector of NormalizedRect at the BATCH_END +# timestamp. +node { + calculator: "EndLoopNormalizedRectCalculator" + input_stream: "ITEM:face_rect_from_detection" + input_stream: "BATCH_END:detections_loop_end_timestamp" + output_stream: "ITERABLE:face_rects_from_detections" +} + +# Performs association between NormalizedRect vector elements from previous +# image and rects based on face detections from the current image. This +# calculator ensures that the output face_rects vector doesn't contain +# overlapping regions based on the specified min_similarity_threshold. +node { + calculator: "AssociationNormRectCalculator" + input_stream: "face_rects_from_detections" + input_stream: "prev_face_rects_from_landmarks" + output_stream: "face_rects" + options: { + [mediapipe.AssociationCalculatorOptions.ext] { + min_similarity_threshold: 0.5 + } + } +} + +# Calculate size of the image. +node { + calculator: "ImagePropertiesCalculator" + input_stream: "IMAGE_GPU:image" + output_stream: "SIZE:image_size" +} + +# Outputs each element of face_rects at a fake timestamp for the rest of the +# graph to process. Clones image and image size packets for each +# single_face_rect at the fake timestamp. At the end of the loop, outputs the +# BATCH_END timestamp for downstream calculators to inform them that all +# elements in the vector have been processed. +node { + calculator: "BeginLoopNormalizedRectCalculator" + input_stream: "ITERABLE:face_rects" + input_stream: "CLONE:0:image" + input_stream: "CLONE:1:image_size" + output_stream: "ITEM:face_rect" + output_stream: "CLONE:0:landmarks_loop_image" + output_stream: "CLONE:1:landmarks_loop_image_size" + output_stream: "BATCH_END:landmarks_loop_end_timestamp" +} + +# Detects face landmarks within specified region of interest of the image. +node { + calculator: "FaceLandmarkSideModelGpu" + input_stream: "IMAGE:landmarks_loop_image" + input_stream: "ROI:face_rect" + input_side_packet: "MODEL:face_landmark_model" + output_stream: "LANDMARKS:face_landmarks" +} + +# Calculates region of interest based on face landmarks, so that can be reused +# for subsequent image. +node { + calculator: "FaceLandmarkLandmarksToRoi" + input_stream: "LANDMARKS:face_landmarks" + input_stream: "IMAGE_SIZE:landmarks_loop_image_size" + output_stream: "ROI:face_rect_from_landmarks" +} + +# Collects a set of landmarks for each face into a vector. Upon receiving the +# BATCH_END timestamp, outputs the vector of landmarks at the BATCH_END +# timestamp. +node { + calculator: "EndLoopNormalizedLandmarkListVectorCalculator" + input_stream: "ITEM:face_landmarks" + input_stream: "BATCH_END:landmarks_loop_end_timestamp" + output_stream: "ITERABLE:multi_face_landmarks" +} + +# Collects a NormalizedRect for each face into a vector. Upon receiving the +# BATCH_END timestamp, outputs the vector of NormalizedRect at the BATCH_END +# timestamp. +node { + calculator: "EndLoopNormalizedRectCalculator" + input_stream: "ITEM:face_rect_from_landmarks" + input_stream: "BATCH_END:landmarks_loop_end_timestamp" + output_stream: "ITERABLE:face_rects_from_landmarks" +} + +# Caches face rects calculated from landmarks, and upon the arrival of the next +# input image, sends out the cached rects with timestamps replaced by that of +# the input image, essentially generating a packet that carries the previous +# face rects. Note that upon the arrival of the very first input image, a +# timestamp bound update occurs to jump start the feedback loop. +node { + calculator: "PreviousLoopbackCalculator" + input_stream: "MAIN:image" + input_stream: "LOOP:face_rects_from_landmarks" + input_stream_info: { + tag_index: "LOOP" + back_edge: true + } + output_stream: "PREV_LOOP:prev_face_rects_from_landmarks" +} diff --git a/mediapipe/modules/hand_landmark/BUILD b/mediapipe/modules/hand_landmark/BUILD index 4dd45e13..83ca097e 100644 --- a/mediapipe/modules/hand_landmark/BUILD +++ b/mediapipe/modules/hand_landmark/BUILD @@ -86,6 +86,26 @@ mediapipe_simple_subgraph( ], ) +mediapipe_simple_subgraph( + name = "hand_landmark_tracking_cpu_image", + graph = "hand_landmark_tracking_cpu_image.pbtxt", + register_as = "HandLandmarkTrackingCpuImage", + deps = [ + ":hand_landmark_tracking_cpu", + "//mediapipe/calculators/util:from_image_calculator", + ], +) + +mediapipe_simple_subgraph( + name = "hand_landmark_tracking_gpu_image", + graph = "hand_landmark_tracking_gpu_image.pbtxt", + register_as = "HandLandmarkTrackingGpuImage", + deps = [ + ":hand_landmark_tracking_gpu", + "//mediapipe/calculators/util:from_image_calculator", + ], +) + mediapipe_simple_subgraph( name = "hand_landmark_tracking_cpu", graph = "hand_landmark_tracking_cpu.pbtxt", @@ -124,7 +144,7 @@ mediapipe_simple_subgraph( graph = "hand_landmark_landmarks_to_roi.pbtxt", register_as = "HandLandmarkLandmarksToRoi", deps = [ - "//mediapipe/calculators/core:split_normalized_landmark_list_calculator", + "//mediapipe/calculators/core:split_landmarks_calculator", "//mediapipe/calculators/util:rect_transformation_calculator", "//mediapipe/modules/hand_landmark/calculators:hand_landmarks_to_rect_calculator", ], diff --git a/mediapipe/modules/hand_landmark/hand_landmark_tracking_cpu_image.pbtxt b/mediapipe/modules/hand_landmark/hand_landmark_tracking_cpu_image.pbtxt new file mode 100644 index 00000000..4d9fc08f --- /dev/null +++ b/mediapipe/modules/hand_landmark/hand_landmark_tracking_cpu_image.pbtxt @@ -0,0 +1,57 @@ +# MediaPipe graph to detect/predict hand landmarks on CPU. +# +# The procedure is done in two steps: +# - locate palms/hands +# - detect landmarks for each palm/hand. +# This graph tries to skip palm detection as much as possible by reusing +# previously detected/predicted landmarks for new images. + +type: "HandLandmarkTrackingCpuImage" + +# Input image. (Image) +input_stream: "IMAGE:image" + +# Max number of hands to detect/track. (int) +input_side_packet: "NUM_HANDS:num_hands" + +# Collection of detected/predicted hands, each represented as a list of +# landmarks. (std::vector) +# NOTE: there will not be an output packet in the LANDMARKS stream for this +# particular timestamp if none of hands detected. However, the MediaPipe +# framework will internally inform the downstream calculators of the absence of +# this packet so that they don't wait for it unnecessarily. +output_stream: "LANDMARKS:multi_hand_landmarks" +# Collection of handedness of the detected hands (i.e. is hand left or right), +# each represented as a Classification proto. +# Note that handedness is determined assuming the input image is mirrored, +# i.e., taken with a front-facing/selfie camera with images flipped +# horizontally. +output_stream: "HANDEDNESS:multi_handedness" + +# Extra outputs (for debugging, for instance). +# Detected palms. (std::vector) +output_stream: "PALM_DETECTIONS:palm_detections" +# Regions of interest calculated based on landmarks. +# (std::vector) +output_stream: "HAND_ROIS_FROM_LANDMARKS:hand_rects" +# Regions of interest calculated based on palm detections. +# (std::vector) +output_stream: "HAND_ROIS_FROM_PALM_DETECTIONS:hand_rects_from_palm_detections" + +# Converts Image to GpuBuffer for HandLandmarkTrackingGpu to consume. +node { + calculator: "FromImageCalculator" + input_stream: "IMAGE:image" + output_stream: "IMAGE_CPU:image_frame" +} + +node { + calculator: "HandLandmarkTrackingCpu" + input_stream: "IMAGE:image_frame" + input_side_packet: "NUM_HANDS:num_hands" + output_stream: "LANDMARKS:multi_hand_landmarks" + output_stream: "HANDEDNESS:multi_handedness" + output_stream: "PALM_DETECTIONS:palm_detections" + output_stream: "HAND_ROIS_FROM_LANDMARKS:hand_rects" + output_stream: "HAND_ROIS_FROM_PALM_DETECTIONS:hand_rects_from_palm_detections" +} diff --git a/mediapipe/modules/hand_landmark/hand_landmark_tracking_gpu_image.pbtxt b/mediapipe/modules/hand_landmark/hand_landmark_tracking_gpu_image.pbtxt new file mode 100644 index 00000000..c580d9d6 --- /dev/null +++ b/mediapipe/modules/hand_landmark/hand_landmark_tracking_gpu_image.pbtxt @@ -0,0 +1,59 @@ +# MediaPipe graph to detect/predict hand landmarks on GPU. +# +# The procedure is done in two steps: +# - locate palms/hands +# - detect landmarks for each palm/hand. +# This graph tries to skip palm detection as much as possible by reusing +# previously detected/predicted landmarks for new images. + +type: "HandLandmarkTrackingGpuImage" + +# Input image. (Image) +input_stream: "IMAGE:image" + +# Max number of hands to detect/track. (int) +input_side_packet: "NUM_HANDS:num_hands" + +# Collection of detected/predicted hands, each represented as a list of +# landmarks. (std::vector) +# NOTE: there will not be an output packet in the LANDMARKS stream for this +# particular timestamp if none of hands detected. However, the MediaPipe +# framework will internally inform the downstream calculators of the absence of +# this packet so that they don't wait for it unnecessarily. +output_stream: "LANDMARKS:multi_hand_landmarks" +# Collection of handedness of the detected hands (i.e. is hand left or right), +# each represented as a Classification proto. +# Note that handedness is determined assuming the input image is mirrored, +# i.e., taken with a front-facing/selfie camera with images flipped +# horizontally. +output_stream: "HANDEDNESS:multi_handedness" + +# The original input image. (Image) +output_stream: "IMAGE:image" +# Extra outputs (for debugging, for instance). +# Detected palms. (std::vector) +output_stream: "PALM_DETECTIONS:palm_detections" +# Regions of interest calculated based on landmarks. +# (std::vector) +output_stream: "HAND_ROIS_FROM_LANDMARKS:hand_rects" +# Regions of interest calculated based on palm detections. +# (std::vector) +output_stream: "HAND_ROIS_FROM_PALM_DETECTIONS:hand_rects_from_palm_detections" + +# Converts Image to GpuBuffer for HandLandmarkTrackingGpu to consume. +node { + calculator: "FromImageCalculator" + input_stream: "IMAGE:image" + output_stream: "IMAGE_GPU:gpu_buffer" +} + +node { + calculator: "HandLandmarkTrackingGpu" + input_stream: "IMAGE:gpu_buffer" + input_side_packet: "NUM_HANDS:num_hands" + output_stream: "LANDMARKS:multi_hand_landmarks" + output_stream: "HANDEDNESS:multi_handedness" + output_stream: "PALM_DETECTIONS:palm_detections" + output_stream: "HAND_ROIS_FROM_LANDMARKS:hand_rects" + output_stream: "HAND_ROIS_FROM_PALM_DETECTIONS:hand_rects_from_palm_detections" +} diff --git a/mediapipe/modules/holistic_landmark/BUILD b/mediapipe/modules/holistic_landmark/BUILD index 0b6e4056..44854c0d 100644 --- a/mediapipe/modules/holistic_landmark/BUILD +++ b/mediapipe/modules/holistic_landmark/BUILD @@ -31,9 +31,9 @@ mediapipe_simple_subgraph( ":face_detection_front_detections_to_roi", ":face_landmarks_from_pose_to_recrop_roi", ":face_tracking", - "//mediapipe/calculators/core:split_normalized_landmark_list_calculator", + "//mediapipe/calculators/core:split_landmarks_calculator", "//mediapipe/calculators/image:image_properties_calculator", - "//mediapipe/modules/face_detection:face_detection_front_by_roi_gpu", + "//mediapipe/modules/face_detection:face_detection_short_range_by_roi_gpu", "//mediapipe/modules/face_landmark:face_landmark_gpu", ], ) @@ -46,9 +46,9 @@ mediapipe_simple_subgraph( ":face_detection_front_detections_to_roi", ":face_landmarks_from_pose_to_recrop_roi", ":face_tracking", - "//mediapipe/calculators/core:split_normalized_landmark_list_calculator", + "//mediapipe/calculators/core:split_landmarks_calculator", "//mediapipe/calculators/image:image_properties_calculator", - "//mediapipe/modules/face_detection:face_detection_front_by_roi_cpu", + "//mediapipe/modules/face_detection:face_detection_short_range_by_roi_cpu", "//mediapipe/modules/face_landmark:face_landmark_cpu", ], ) @@ -131,7 +131,7 @@ mediapipe_simple_subgraph( graph = "hand_landmarks_to_roi.pbtxt", register_as = "HandLandmarksToRoi", deps = [ - "//mediapipe/calculators/core:split_normalized_landmark_list_calculator", + "//mediapipe/calculators/core:split_landmarks_calculator", "//mediapipe/calculators/util:rect_transformation_calculator", "//mediapipe/modules/hand_landmark/calculators:hand_landmarks_to_rect_calculator", ], @@ -191,7 +191,7 @@ mediapipe_simple_subgraph( deps = [ "//mediapipe/calculators/core:constant_side_packet_calculator", "//mediapipe/calculators/core:side_packet_to_stream_calculator", - "//mediapipe/calculators/core:split_normalized_landmark_list_calculator", + "//mediapipe/calculators/core:split_landmarks_calculator", "//mediapipe/calculators/util:set_landmark_visibility_calculator", ], ) @@ -202,7 +202,7 @@ mediapipe_simple_subgraph( register_as = "HandLandmarksLeftAndRightGpu", deps = [ ":hand_landmarks_from_pose_gpu", - "//mediapipe/calculators/core:split_normalized_landmark_list_calculator", + "//mediapipe/calculators/core:split_landmarks_calculator", ], ) @@ -212,7 +212,7 @@ mediapipe_simple_subgraph( register_as = "HandLandmarksLeftAndRightCpu", deps = [ ":hand_landmarks_from_pose_cpu", - "//mediapipe/calculators/core:split_normalized_landmark_list_calculator", + "//mediapipe/calculators/core:split_landmarks_calculator", ], ) @@ -232,7 +232,7 @@ mediapipe_simple_subgraph( graph = "hand_visibility_from_hand_landmarks_from_pose.pbtxt", register_as = "HandVisibilityFromHandLandmarksFromPose", deps = [ - "//mediapipe/calculators/core:split_normalized_landmark_list_calculator", + "//mediapipe/calculators/core:split_landmarks_calculator", "//mediapipe/calculators/util:landmark_visibility_calculator", "//mediapipe/calculators/util:thresholding_calculator", ], diff --git a/mediapipe/modules/holistic_landmark/face_detection_front_detections_to_roi.pbtxt b/mediapipe/modules/holistic_landmark/face_detection_front_detections_to_roi.pbtxt index 9d119073..7d9fa9e2 100644 --- a/mediapipe/modules/holistic_landmark/face_detection_front_detections_to_roi.pbtxt +++ b/mediapipe/modules/holistic_landmark/face_detection_front_detections_to_roi.pbtxt @@ -1,4 +1,4 @@ -# Calculates ROI from detections provided by `face_detection_front.tflite` +# Calculates ROI from detections provided by `face_detection_short_range.tflite` # model. type: "FaceDetectionFrontDetectionsToRoi" diff --git a/mediapipe/modules/holistic_landmark/face_landmarks_from_pose_cpu.pbtxt b/mediapipe/modules/holistic_landmark/face_landmarks_from_pose_cpu.pbtxt index e8b18c5a..e1a25e90 100644 --- a/mediapipe/modules/holistic_landmark/face_landmarks_from_pose_cpu.pbtxt +++ b/mediapipe/modules/holistic_landmark/face_landmarks_from_pose_cpu.pbtxt @@ -43,7 +43,7 @@ node { # to refine face ROI for further landmark detection as ROI calculated from # pose landmarks may be inaccurate. node { - calculator: "FaceDetectionFrontByRoiCpu" + calculator: "FaceDetectionShortRangeByRoiCpu" input_stream: "IMAGE:input_video" input_stream: "ROI:face_roi_from_pose" output_stream: "DETECTIONS:face_detections" @@ -51,7 +51,7 @@ node { # Calculates refined face ROI. node { - calculator: "FaceDetectionFrontDetectionsToRoi", + calculator: "FaceDetectionFrontDetectionsToRoi" input_stream: "DETECTIONS:face_detections" input_stream: "IMAGE_SIZE:image_size" output_stream: "ROI:face_roi_from_detection" diff --git a/mediapipe/modules/holistic_landmark/face_landmarks_from_pose_gpu.pbtxt b/mediapipe/modules/holistic_landmark/face_landmarks_from_pose_gpu.pbtxt index f5b2f426..872526b0 100644 --- a/mediapipe/modules/holistic_landmark/face_landmarks_from_pose_gpu.pbtxt +++ b/mediapipe/modules/holistic_landmark/face_landmarks_from_pose_gpu.pbtxt @@ -43,7 +43,7 @@ node { # to refine face ROI for further landmark detection as ROI calculated from # pose landmarks may be inaccurate. node { - calculator: "FaceDetectionFrontByRoiGpu" + calculator: "FaceDetectionShortRangeByRoiGpu" input_stream: "IMAGE:input_video" input_stream: "ROI:face_roi_from_pose" output_stream: "DETECTIONS:face_detections" @@ -51,7 +51,7 @@ node { # Calculates refined face ROI. node { - calculator: "FaceDetectionFrontDetectionsToRoi", + calculator: "FaceDetectionFrontDetectionsToRoi" input_stream: "DETECTIONS:face_detections" input_stream: "IMAGE_SIZE:image_size" output_stream: "ROI:face_roi_from_detection" diff --git a/mediapipe/modules/holistic_landmark/holistic_landmark_cpu.pbtxt b/mediapipe/modules/holistic_landmark/holistic_landmark_cpu.pbtxt index fa1d5c26..84b5b2b7 100644 --- a/mediapipe/modules/holistic_landmark/holistic_landmark_cpu.pbtxt +++ b/mediapipe/modules/holistic_landmark/holistic_landmark_cpu.pbtxt @@ -1,8 +1,8 @@ # Predicts pose + left/right hand + face landmarks. # # It is required that: -# - "face_detection_front.tflite" is available at -# "mediapipe/modules/face_detection/face_detection_front.tflite" +# - "face_detection_short_range.tflite" is available at +# "mediapipe/modules/face_detection/face_detection_short_range.tflite" # # - "face_landmark.tflite" is available at # "mediapipe/modules/face_landmark/face_landmark.tflite" @@ -62,6 +62,8 @@ input_side_packet: "SMOOTH_LANDMARKS:smooth_landmarks" # Pose landmarks. (NormalizedLandmarkList) # 33 pose landmarks. output_stream: "POSE_LANDMARKS:pose_landmarks" +# 33 pose world landmarks. (LandmarkList) +output_stream: "WORLD_LANDMARKS:pose_world_landmarks" # 21 left hand landmarks. (NormalizedLandmarkList) output_stream: "LEFT_HAND_LANDMARKS:left_hand_landmarks" # 21 right hand landmarks. (NormalizedLandmarkList) @@ -80,6 +82,7 @@ node { input_side_packet: "MODEL_COMPLEXITY:model_complexity" input_side_packet: "SMOOTH_LANDMARKS:smooth_landmarks" output_stream: "LANDMARKS:pose_landmarks" + output_stream: "WORLD_LANDMARKS:pose_world_landmarks" output_stream: "ROI_FROM_LANDMARKS:pose_landmarks_roi" output_stream: "DETECTION:pose_detection" } diff --git a/mediapipe/modules/holistic_landmark/holistic_landmark_gpu.pbtxt b/mediapipe/modules/holistic_landmark/holistic_landmark_gpu.pbtxt index 1f6fa63d..fc227d00 100644 --- a/mediapipe/modules/holistic_landmark/holistic_landmark_gpu.pbtxt +++ b/mediapipe/modules/holistic_landmark/holistic_landmark_gpu.pbtxt @@ -1,8 +1,8 @@ # Predicts pose + left/right hand + face landmarks. # # It is required that: -# - "face_detection_front.tflite" is available at -# "mediapipe/modules/face_detection/face_detection_front.tflite" +# - "face_detection_short_range.tflite" is available at +# "mediapipe/modules/face_detection/face_detection_short_range.tflite" # # - "face_landmark.tflite" is available at # "mediapipe/modules/face_landmark/face_landmark.tflite" @@ -62,6 +62,8 @@ input_side_packet: "SMOOTH_LANDMARKS:smooth_landmarks" # Pose landmarks. (NormalizedLandmarkList) # 33 pose landmarks. output_stream: "POSE_LANDMARKS:pose_landmarks" +# 33 pose world landmarks. (LandmarkList) +output_stream: "WORLD_LANDMARKS:pose_world_landmarks" # 21 left hand landmarks. (NormalizedLandmarkList) output_stream: "LEFT_HAND_LANDMARKS:left_hand_landmarks" # 21 right hand landmarks. (NormalizedLandmarkList) @@ -80,6 +82,7 @@ node { input_side_packet: "MODEL_COMPLEXITY:model_complexity" input_side_packet: "SMOOTH_LANDMARKS:smooth_landmarks" output_stream: "LANDMARKS:pose_landmarks" + output_stream: "WORLD_LANDMARKS:pose_world_landmarks" output_stream: "ROI_FROM_LANDMARKS:pose_landmarks_roi" output_stream: "DETECTION:pose_detection" } diff --git a/mediapipe/modules/pose_detection/pose_detection.tflite b/mediapipe/modules/pose_detection/pose_detection.tflite index 57f89be5..77bd6fae 100755 Binary files a/mediapipe/modules/pose_detection/pose_detection.tflite and b/mediapipe/modules/pose_detection/pose_detection.tflite differ diff --git a/mediapipe/modules/pose_landmark/BUILD b/mediapipe/modules/pose_landmark/BUILD index f38b2040..3b2a6f4b 100644 --- a/mediapipe/modules/pose_landmark/BUILD +++ b/mediapipe/modules/pose_landmark/BUILD @@ -38,18 +38,10 @@ mediapipe_simple_subgraph( graph = "pose_landmark_by_roi_gpu.pbtxt", register_as = "PoseLandmarkByRoiGpu", deps = [ + ":pose_landmark_by_roi_post_processing", ":pose_landmark_model_loader", - "//mediapipe/calculators/core:gate_calculator", - "//mediapipe/calculators/core:split_normalized_landmark_list_calculator", - "//mediapipe/calculators/core:split_vector_calculator", "//mediapipe/calculators/tensor:image_to_tensor_calculator", "//mediapipe/calculators/tensor:inference_calculator", - "//mediapipe/calculators/tensor:tensors_to_floats_calculator", - "//mediapipe/calculators/tensor:tensors_to_landmarks_calculator", - "//mediapipe/calculators/util:landmark_letterbox_removal_calculator", - "//mediapipe/calculators/util:landmark_projection_calculator", - "//mediapipe/calculators/util:refine_landmarks_from_heatmap_calculator", - "//mediapipe/calculators/util:thresholding_calculator", ], ) @@ -58,18 +50,29 @@ mediapipe_simple_subgraph( graph = "pose_landmark_by_roi_cpu.pbtxt", register_as = "PoseLandmarkByRoiCpu", deps = [ + ":pose_landmark_by_roi_post_processing", ":pose_landmark_model_loader", - "//mediapipe/calculators/core:gate_calculator", - "//mediapipe/calculators/core:split_normalized_landmark_list_calculator", - "//mediapipe/calculators/core:split_vector_calculator", "//mediapipe/calculators/tensor:image_to_tensor_calculator", "//mediapipe/calculators/tensor:inference_calculator", + ], +) + +mediapipe_simple_subgraph( + name = "pose_landmark_by_roi_post_processing", + graph = "pose_landmark_by_roi_post_processing.pbtxt", + register_as = "PoseLandmarkByRoiPostProcessing", + deps = [ + "//mediapipe/calculators/core:gate_calculator", + "//mediapipe/calculators/core:split_landmarks_calculator", + "//mediapipe/calculators/core:split_vector_calculator", "//mediapipe/calculators/tensor:tensors_to_floats_calculator", "//mediapipe/calculators/tensor:tensors_to_landmarks_calculator", "//mediapipe/calculators/util:landmark_letterbox_removal_calculator", "//mediapipe/calculators/util:landmark_projection_calculator", "//mediapipe/calculators/util:refine_landmarks_from_heatmap_calculator", "//mediapipe/calculators/util:thresholding_calculator", + "//mediapipe/calculators/util:visibility_copy_calculator", + "//mediapipe/calculators/util:world_landmark_projection_calculator", ], ) diff --git a/mediapipe/modules/pose_landmark/pose_landmark_by_roi_cpu.pbtxt b/mediapipe/modules/pose_landmark/pose_landmark_by_roi_cpu.pbtxt index d98ad4a4..8c4c6f43 100644 --- a/mediapipe/modules/pose_landmark/pose_landmark_by_roi_cpu.pbtxt +++ b/mediapipe/modules/pose_landmark/pose_landmark_by_roi_cpu.pbtxt @@ -77,6 +77,14 @@ output_stream: "LANDMARKS:landmarks" # (NormalizedLandmarkList) output_stream: "AUXILIARY_LANDMARKS:auxiliary_landmarks" +# Pose world landmarks within the given ROI. (LandmarkList) +# World landmarks are real-world 3D coordinates in meters with the origin at the +# center between hips. WORLD_LANDMARKS shares the same landmark topology as +# LANDMARKS. However, LANDMARKS provides coordinates (in pixels) of a 3D object +# projected onto the 2D image surface, while WORLD_LANDMARKS provides +# coordinates (in meters) of the 3D object itself. +output_stream: "WORLD_LANDMARKS:pose_world_landmarks" + # Transforms the input image into a 256x256 tensor while keeping the aspect # ratio (what is expected by the corresponding model), resulting in potential # letterboxing in the transformed image. @@ -119,113 +127,12 @@ node { } } -# Splits a vector of TFLite tensors to multiple vectors according to the ranges -# specified in option. node { - calculator: "SplitTensorVectorCalculator" - input_stream: "output_tensors" - output_stream: "landmark_tensors" - output_stream: "pose_flag_tensor" - output_stream: "heatmap_tensor" - options: { - [mediapipe.SplitVectorCalculatorOptions.ext] { - ranges: { begin: 0 end: 1 } - ranges: { begin: 1 end: 2 } - ranges: { begin: 3 end: 4 } - } - } -} - -# Converts the pose-flag tensor into a float that represents the confidence -# score of pose presence. -node { - calculator: "TensorsToFloatsCalculator" - input_stream: "TENSORS:pose_flag_tensor" - output_stream: "FLOAT:pose_presence_score" -} - -# Applies a threshold to the confidence score to determine whether a pose is -# present. -node { - calculator: "ThresholdingCalculator" - input_stream: "FLOAT:pose_presence_score" - output_stream: "FLAG:pose_presence" - options: { - [mediapipe.ThresholdingCalculatorOptions.ext] { - threshold: 0.5 - } - } -} - -# Drops landmark tensors if pose is not present. -node { - calculator: "GateCalculator" - input_stream: "landmark_tensors" - input_stream: "ALLOW:pose_presence" - output_stream: "ensured_landmark_tensors" -} - -# Decodes the landmark tensors into a vector of landmarks, where the landmark -# coordinates are normalized by the size of the input image to the model. -node { - calculator: "TensorsToLandmarksCalculator" - input_stream: "TENSORS:ensured_landmark_tensors" - output_stream: "NORM_LANDMARKS:raw_landmarks" - options: { - [mediapipe.TensorsToLandmarksCalculatorOptions.ext] { - num_landmarks: 39 - input_image_width: 256 - input_image_height: 256 - visibility_activation: SIGMOID - presence_activation: SIGMOID - } - } -} - -# Refines landmarks with the heatmap tensor. -node { - calculator: "RefineLandmarksFromHeatmapCalculator" - input_stream: "NORM_LANDMARKS:raw_landmarks" - input_stream: "TENSORS:heatmap_tensor" - output_stream: "NORM_LANDMARKS:refined_landmarks" - options: { - [mediapipe.RefineLandmarksFromHeatmapCalculatorOptions.ext] { - kernel_size: 7 - } - } -} - -# Adjusts landmarks (already normalized to [0.f, 1.f]) on the letterboxed pose -# image (after image transformation with the FIT scale mode) to the -# corresponding locations on the same image with the letterbox removed (pose -# image before image transformation). -node { - calculator: "LandmarkLetterboxRemovalCalculator" - input_stream: "LANDMARKS:refined_landmarks" + calculator: "PoseLandmarkByRoiPostProcessing" + input_stream: "TENSORS:output_tensors" + input_stream: "ROI:roi" input_stream: "LETTERBOX_PADDING:letterbox_padding" - output_stream: "LANDMARKS:adjusted_landmarks" -} - -# Projects the landmarks from the cropped pose image to the corresponding -# locations on the full image before cropping (input to the graph). -node { - calculator: "LandmarkProjectionCalculator" - input_stream: "NORM_LANDMARKS:adjusted_landmarks" - input_stream: "NORM_RECT:roi" - output_stream: "NORM_LANDMARKS:all_landmarks" -} - -# Splits the landmarks into two sets: the actual pose landmarks and the -# auxiliary landmarks. -node { - calculator: "SplitNormalizedLandmarkListCalculator" - input_stream: "all_landmarks" - output_stream: "landmarks" - output_stream: "auxiliary_landmarks" - options: { - [mediapipe.SplitVectorCalculatorOptions.ext] { - ranges: { begin: 0 end: 33 } - ranges: { begin: 33 end: 35 } - } - } + output_stream: "LANDMARKS:landmarks" + output_stream: "AUXILIARY_LANDMARKS:auxiliary_landmarks" + output_stream: "WORLD_LANDMARKS:pose_world_landmarks" } diff --git a/mediapipe/modules/pose_landmark/pose_landmark_by_roi_gpu.pbtxt b/mediapipe/modules/pose_landmark/pose_landmark_by_roi_gpu.pbtxt index 7cb87d0e..9da60f40 100644 --- a/mediapipe/modules/pose_landmark/pose_landmark_by_roi_gpu.pbtxt +++ b/mediapipe/modules/pose_landmark/pose_landmark_by_roi_gpu.pbtxt @@ -77,6 +77,14 @@ output_stream: "LANDMARKS:landmarks" # (NormalizedLandmarkList) output_stream: "AUXILIARY_LANDMARKS:auxiliary_landmarks" +# Pose world landmarks within the given ROI. (LandmarkList) +# World landmarks are real-world 3D coordinates in meters with the origin at the +# center between hips. WORLD_LANDMARKS shares the same landmark topology as +# LANDMARKS. However, LANDMARKS provides coordinates (in pixels) of a 3D object +# projected onto the 2D image surface, while WORLD_LANDMARKS provides +# coordinates (in meters) of the 3D object itself. +output_stream: "WORLD_LANDMARKS:pose_world_landmarks" + # Transforms the input image into a 256x256 tensor while keeping the aspect # ratio (what is expected by the corresponding model), resulting in potential # letterboxing in the transformed image. @@ -113,124 +121,14 @@ node { input_side_packet: "MODEL:model" input_stream: "TENSORS:input_tensors" output_stream: "TENSORS:output_tensors" - options: { - [mediapipe.InferenceCalculatorOptions.ext] { - delegate { - gpu { - allow_precision_loss: false - } - } - } - } } -# Splits a vector of TFLite tensors to multiple vectors according to the ranges -# specified in option. node { - calculator: "SplitTensorVectorCalculator" - input_stream: "output_tensors" - output_stream: "landmark_tensors" - output_stream: "pose_flag_tensor" - output_stream: "heatmap_tensor" - options: { - [mediapipe.SplitVectorCalculatorOptions.ext] { - ranges: { begin: 0 end: 1 } - ranges: { begin: 1 end: 2 } - ranges: { begin: 3 end: 4 } - } - } -} - -# Converts the pose-flag tensor into a float that represents the confidence -# score of pose presence. -node { - calculator: "TensorsToFloatsCalculator" - input_stream: "TENSORS:pose_flag_tensor" - output_stream: "FLOAT:pose_presence_score" -} - -# Applies a threshold to the confidence score to determine whether a pose is -# present. -node { - calculator: "ThresholdingCalculator" - input_stream: "FLOAT:pose_presence_score" - output_stream: "FLAG:pose_presence" - options: { - [mediapipe.ThresholdingCalculatorOptions.ext] { - threshold: 0.5 - } - } -} - -# Drops landmark tensors if pose is not present. -node { - calculator: "GateCalculator" - input_stream: "landmark_tensors" - input_stream: "ALLOW:pose_presence" - output_stream: "ensured_landmark_tensors" -} - -# Decodes the landmark tensors into a vector of landmarks, where the landmark -# coordinates are normalized by the size of the input image to the model. -node { - calculator: "TensorsToLandmarksCalculator" - input_stream: "TENSORS:ensured_landmark_tensors" - output_stream: "NORM_LANDMARKS:raw_landmarks" - options: { - [mediapipe.TensorsToLandmarksCalculatorOptions.ext] { - num_landmarks: 39 - input_image_width: 256 - input_image_height: 256 - visibility_activation: SIGMOID - presence_activation: SIGMOID - } - } -} - -# Refines landmarks with the heatmap tensor. -node { - calculator: "RefineLandmarksFromHeatmapCalculator" - input_stream: "NORM_LANDMARKS:raw_landmarks" - input_stream: "TENSORS:heatmap_tensor" - output_stream: "NORM_LANDMARKS:refined_landmarks" - options: { - [mediapipe.RefineLandmarksFromHeatmapCalculatorOptions.ext] { - kernel_size: 7 - } - } -} - -# Adjusts landmarks (already normalized to [0.f, 1.f]) on the letterboxed pose -# image (after image transformation with the FIT scale mode) to the -# corresponding locations on the same image with the letterbox removed (pose -# image before image transformation). -node { - calculator: "LandmarkLetterboxRemovalCalculator" - input_stream: "LANDMARKS:refined_landmarks" + calculator: "PoseLandmarkByRoiPostProcessing" + input_stream: "TENSORS:output_tensors" + input_stream: "ROI:roi" input_stream: "LETTERBOX_PADDING:letterbox_padding" - output_stream: "LANDMARKS:adjusted_landmarks" -} - -# Projects the landmarks from the cropped pose image to the corresponding -# locations on the full image before cropping (input to the graph). -node { - calculator: "LandmarkProjectionCalculator" - input_stream: "NORM_LANDMARKS:adjusted_landmarks" - input_stream: "NORM_RECT:roi" - output_stream: "NORM_LANDMARKS:all_landmarks" -} - -# Splits the landmarks into two sets: the actual pose landmarks and the -# auxiliary landmarks. -node { - calculator: "SplitNormalizedLandmarkListCalculator" - input_stream: "all_landmarks" - output_stream: "landmarks" - output_stream: "auxiliary_landmarks" - options: { - [mediapipe.SplitVectorCalculatorOptions.ext] { - ranges: { begin: 0 end: 33 } - ranges: { begin: 33 end: 35 } - } - } + output_stream: "LANDMARKS:landmarks" + output_stream: "AUXILIARY_LANDMARKS:auxiliary_landmarks" + output_stream: "WORLD_LANDMARKS:pose_world_landmarks" } diff --git a/mediapipe/modules/pose_landmark/pose_landmark_by_roi_post_processing.pbtxt b/mediapipe/modules/pose_landmark/pose_landmark_by_roi_post_processing.pbtxt new file mode 100644 index 00000000..a3a79de3 --- /dev/null +++ b/mediapipe/modules/pose_landmark/pose_landmark_by_roi_post_processing.pbtxt @@ -0,0 +1,246 @@ +# MediaPipe graph performing tensor post processing to detect/predict pose +# landmarks. +# +# EXAMPLE: +# node { +# calculator: "PoseLandmarkByRoiPostProcessing" +# input_stream: "TENSORS:tensors" +# input_stream: "ROI:roi" +# input_stream: "LETTERBOX_PADDING:letterbox_padding" +# output_stream: "LANDMARKS:landmarks" +# output_stream: "AUXILIARY_LANDMARKS:auxiliary_landmarks" +# } + +type: "PoseLandmarkByRoiPostProcessing" + +# Tensors from model inference of pose_landmark_{full|heavy|lite}.tflite. +# (std::vector) +input_stream: "TENSORS:tensors" + +# ROI (region of interest) within a given image, where only the ROI is used as +# input to model inference but the resulting pose landmarks are to be reported +# in coordinates w.r.t. the entire image. (NormalizedRect) +input_stream: "ROI:roi" + +# An array representing the letterbox padding from the 4 sides, i.e., +# [left, top, right, bottom], padded around the ROI above before it is used as +# input to model inference. The padding is normalized to [0.f, 1.f] relatvie to +# the ROI dimensions. (std::array) +input_stream: "LETTERBOX_PADDING:letterbox_padding" + +# Pose landmarks within the given ROI. (NormalizedLandmarkList) +# We have 33 landmarks (see pose_landmark_topology.svg) and there are other +# auxiliary key points. +# 0 - nose +# 1 - left eye (inner) +# 2 - left eye +# 3 - left eye (outer) +# 4 - right eye (inner) +# 5 - right eye +# 6 - right eye (outer) +# 7 - left ear +# 8 - right ear +# 9 - mouth (left) +# 10 - mouth (right) +# 11 - left shoulder +# 12 - right shoulder +# 13 - left elbow +# 14 - right elbow +# 15 - left wrist +# 16 - right wrist +# 17 - left pinky +# 18 - right pinky +# 19 - left index +# 20 - right index +# 21 - left thumb +# 22 - right thumb +# 23 - left hip +# 24 - right hip +# 25 - left knee +# 26 - right knee +# 27 - left ankle +# 28 - right ankle +# 29 - left heel +# 30 - right heel +# 31 - left foot index +# 32 - right foot index +# +# NOTE: if a pose is not present within the given ROI, for this particular +# timestamp there will not be an output packet in the LANDMARKS stream. However, +# the MediaPipe framework will internally inform the downstream calculators of +# the absence of this packet so that they don't wait for it unnecessarily. +output_stream: "LANDMARKS:landmarks" +# Auxiliary landmarks for deriving the ROI in the subsequent image. +# (NormalizedLandmarkList) +output_stream: "AUXILIARY_LANDMARKS:auxiliary_landmarks" + +# Pose world landmarks within the given ROI. (LandmarkList) +# World landmarks are real-world 3D coordinates in meters with the origin at the +# center between hips. WORLD_LANDMARKS shares the same landmark topology as +# LANDMARKS. However, LANDMARKS provides coordinates (in pixels) of a 3D object +# projected onto the 2D image surface, while WORLD_LANDMARKS provides +# coordinates (in meters) of the 3D object itself. +output_stream: "WORLD_LANDMARKS:world_landmarks" + +# Splits a vector of TFLite tensors to multiple vectors according to the ranges +# specified in option. +node { + calculator: "SplitTensorVectorCalculator" + input_stream: "tensors" + output_stream: "landmark_tensors" + output_stream: "pose_flag_tensor" + output_stream: "heatmap_tensor" + output_stream: "world_landmark_tensors" + options: { + [mediapipe.SplitVectorCalculatorOptions.ext] { + ranges: { begin: 0 end: 1 } + ranges: { begin: 1 end: 2 } + ranges: { begin: 3 end: 4 } + ranges: { begin: 4 end: 5 } + } + } +} + +# Converts the pose-flag tensor into a float that represents the confidence +# score of pose presence. +node { + calculator: "TensorsToFloatsCalculator" + input_stream: "TENSORS:pose_flag_tensor" + output_stream: "FLOAT:pose_presence_score" +} + +# Applies a threshold to the confidence score to determine whether a pose is +# present. +node { + calculator: "ThresholdingCalculator" + input_stream: "FLOAT:pose_presence_score" + output_stream: "FLAG:pose_presence" + options: { + [mediapipe.ThresholdingCalculatorOptions.ext] { + threshold: 0.5 + } + } +} + +# Drops landmark tensors if pose is not present. +node { + calculator: "GateCalculator" + input_stream: "landmark_tensors" + input_stream: "world_landmark_tensors" + input_stream: "ALLOW:pose_presence" + output_stream: "ensured_landmark_tensors" + output_stream: "ensured_world_landmark_tensors" +} + +# Decodes the landmark tensors into a vector of landmarks, where the landmark +# coordinates are normalized by the size of the input image to the model. +node { + calculator: "TensorsToLandmarksCalculator" + input_stream: "TENSORS:ensured_landmark_tensors" + output_stream: "NORM_LANDMARKS:raw_landmarks" + options: { + [mediapipe.TensorsToLandmarksCalculatorOptions.ext] { + num_landmarks: 39 + input_image_width: 256 + input_image_height: 256 + visibility_activation: SIGMOID + presence_activation: SIGMOID + } + } +} + +# Refines landmarks with the heatmap tensor. +node { + calculator: "RefineLandmarksFromHeatmapCalculator" + input_stream: "NORM_LANDMARKS:raw_landmarks" + input_stream: "TENSORS:heatmap_tensor" + output_stream: "NORM_LANDMARKS:refined_landmarks" + options: { + [mediapipe.RefineLandmarksFromHeatmapCalculatorOptions.ext] { + kernel_size: 7 + } + } +} + +# Adjusts landmarks (already normalized to [0.f, 1.f]) on the letterboxed pose +# image (after image transformation with the FIT scale mode) to the +# corresponding locations on the same image with the letterbox removed (pose +# image before image transformation). +node { + calculator: "LandmarkLetterboxRemovalCalculator" + input_stream: "LANDMARKS:refined_landmarks" + input_stream: "LETTERBOX_PADDING:letterbox_padding" + output_stream: "LANDMARKS:adjusted_landmarks" +} + +# Projects the landmarks from the cropped pose image to the corresponding +# locations on the full image before cropping. +node { + calculator: "LandmarkProjectionCalculator" + input_stream: "NORM_LANDMARKS:adjusted_landmarks" + input_stream: "NORM_RECT:roi" + output_stream: "NORM_LANDMARKS:all_landmarks" +} + +# Splits the landmarks into two sets: the actual pose landmarks and the +# auxiliary landmarks. +node { + calculator: "SplitNormalizedLandmarkListCalculator" + input_stream: "all_landmarks" + output_stream: "landmarks" + output_stream: "auxiliary_landmarks" + options: { + [mediapipe.SplitVectorCalculatorOptions.ext] { + ranges: { begin: 0 end: 33 } + ranges: { begin: 33 end: 35 } + } + } +} + +# Decodes the world landmark tensors into a vector of world landmarks. +node { + calculator: "TensorsToLandmarksCalculator" + input_stream: "TENSORS:ensured_world_landmark_tensors" + output_stream: "LANDMARKS:raw_world_landmarks" + options: { + [mediapipe.TensorsToLandmarksCalculatorOptions.ext] { + num_landmarks: 39 + } + } +} + +# Reuses the visibility and presence field in pose landmarks for the world +# landmarks. +node { + calculator: "VisibilityCopyCalculator" + input_stream: "NORM_LANDMARKS_FROM:raw_landmarks" + input_stream: "LANDMARKS_TO:raw_world_landmarks" + output_stream: "LANDMARKS_TO:raw_world_landmarks_with_visibility" + options: { + [mediapipe.VisibilityCopyCalculatorOptions.ext] { + copy_visibility: true + copy_presence: true + } + } +} + +# Projects the world landmarks from the cropped pose image to the corresponding +# locations on the full image before cropping. +node { + calculator: "WorldLandmarkProjectionCalculator" + input_stream: "LANDMARKS:raw_world_landmarks_with_visibility" + input_stream: "NORM_RECT:roi" + output_stream: "LANDMARKS:all_world_landmarks" +} + +# Takes only actual world landmarks. +node { + calculator: "SplitLandmarkListCalculator" + input_stream: "all_world_landmarks" + output_stream: "world_landmarks" + options: { + [mediapipe.SplitVectorCalculatorOptions.ext] { + ranges: { begin: 0 end: 33 } + } + } +} diff --git a/mediapipe/modules/pose_landmark/pose_landmark_cpu.pbtxt b/mediapipe/modules/pose_landmark/pose_landmark_cpu.pbtxt index e90f2961..9f363124 100644 --- a/mediapipe/modules/pose_landmark/pose_landmark_cpu.pbtxt +++ b/mediapipe/modules/pose_landmark/pose_landmark_cpu.pbtxt @@ -37,7 +37,7 @@ input_side_packet: "SMOOTH_LANDMARKS:smooth_landmarks" # functions as set to 1. (int) input_side_packet: "MODEL_COMPLEXITY:model_complexity" -# Pose landmarks within the given ROI. (NormalizedLandmarkList) +# Pose landmarks. (NormalizedLandmarkList) # We have 33 landmarks (see pose_landmark_topology.svg), and there are other # auxiliary key points. # 0 - nose @@ -80,6 +80,14 @@ input_side_packet: "MODEL_COMPLEXITY:model_complexity" # the absence of this packet so that they don't wait for it unnecessarily. output_stream: "LANDMARKS:pose_landmarks" +# Pose world landmarks. (LandmarkList) +# World landmarks are real-world 3D coordinates in meters with the origin at the +# center between hips. WORLD_LANDMARKS shares the same landmark topology as +# LANDMARKS. However, LANDMARKS provides coordinates (in pixels) of a 3D object +# projected onto the 2D image surface, while WORLD_LANDMARKS provides +# coordinates (in meters) of the 3D object itself. +output_stream: "WORLD_LANDMARKS:pose_world_landmarks" + # Extra outputs (for debugging, for instance). # Detected poses. (Detection) output_stream: "DETECTION:pose_detection" @@ -185,6 +193,7 @@ node { input_stream: "ROI:pose_rect" output_stream: "LANDMARKS:unfiltered_pose_landmarks" output_stream: "AUXILIARY_LANDMARKS:unfiltered_auxiliary_landmarks" + output_stream: "WORLD_LANDMARKS:unfiltered_world_landmarks" } # Smoothes landmarks to reduce jitter. @@ -194,8 +203,10 @@ node { input_stream: "IMAGE_SIZE:image_size" input_stream: "NORM_LANDMARKS:unfiltered_pose_landmarks" input_stream: "AUX_NORM_LANDMARKS:unfiltered_auxiliary_landmarks" + input_stream: "WORLD_LANDMARKS:unfiltered_world_landmarks" output_stream: "FILTERED_NORM_LANDMARKS:pose_landmarks" output_stream: "FILTERED_AUX_NORM_LANDMARKS:auxiliary_landmarks" + output_stream: "FILTERED_WORLD_LANDMARKS:pose_world_landmarks" } # Calculates region of interest based on the auxiliary landmarks, to be used in diff --git a/mediapipe/modules/pose_landmark/pose_landmark_filtering.pbtxt b/mediapipe/modules/pose_landmark/pose_landmark_filtering.pbtxt index 2560dda7..bb3665fe 100644 --- a/mediapipe/modules/pose_landmark/pose_landmark_filtering.pbtxt +++ b/mediapipe/modules/pose_landmark/pose_landmark_filtering.pbtxt @@ -8,8 +8,10 @@ # input_stream: "IMAGE_SIZE:image_size" # input_stream: "NORM_LANDMARKS:landmarks" # input_stream: "AUX_NORM_LANDMARKS:aux_landmarks" +# input_stream: "WORLD_LANDMARKS:world_landmarks" # output_stream: "FILTERED_NORM_LANDMARKS:filtered_landmarks" # output_stream: "FILTERED_AUX_NORM_LANDMARKS:filtered_aux_landmarks" +# output_stream: "FILTERED_WORLD_LANDMARKS:filtered_world_landmarks" # } type: "PoseLandmarkFiltering" @@ -20,14 +22,18 @@ input_side_packet: "ENABLE:enable" # Size of the image (width & height) where the landmarks are estimated from. # (std::pair) input_stream: "IMAGE_SIZE:image_size" -# Normalized landmarks. (NormalizedRect) +# Normalized landmarks. (NormalizedLandmarkList) input_stream: "NORM_LANDMARKS:landmarks" -# Auxiliary set of normalized landmarks. (NormalizedRect) +# Auxiliary set of normalized landmarks. (NormalizedLandmarkList) input_stream: "AUX_NORM_LANDMARKS:aux_landmarks" -# Filtered normalized landmarks. (NormalizedRect) +# World landmarks. (LandmarkList) +input_stream: "WORLD_LANDMARKS:world_landmarks" +# Filtered normalized landmarks. (NormalizedLandmarkList) output_stream: "FILTERED_NORM_LANDMARKS:filtered_landmarks" -# Filtered auxiliary set of normalized landmarks. (NormalizedRect) +# Filtered auxiliary set of normalized landmarks. (NormalizedLandmarkList) output_stream: "FILTERED_AUX_NORM_LANDMARKS:filtered_aux_landmarks" +# Filtered world landmarks. (LandmarkList) +output_stream: "FILTERED_WORLD_LANDMARKS:filtered_world_landmarks" # Converts landmarks to a detection that tightly encloses all landmarks. node { @@ -124,7 +130,81 @@ node { } } -# Smoothes pose landmark visibilities to reduce jitter. +# Smoothes world landmark visibilities to reduce jitter. +node { + calculator: "SwitchContainer" + input_side_packet: "ENABLE:enable" + input_stream: "LANDMARKS:world_landmarks" + output_stream: "FILTERED_LANDMARKS:filtered_world_visibility" + options: { + [mediapipe.SwitchContainerOptions.ext] { + enable: true + contained_node: { + calculator: "VisibilitySmoothingCalculator" + options: { + [mediapipe.VisibilitySmoothingCalculatorOptions.ext] { + no_filter: {} + } + } + } + contained_node: { + calculator: "VisibilitySmoothingCalculator" + options: { + [mediapipe.VisibilitySmoothingCalculatorOptions.ext] { + low_pass_filter { + alpha: 0.1 + } + } + } + } + } + } +} + +# Smoothes world landmark coordinates to reduce jitter. +node { + calculator: "SwitchContainer" + input_side_packet: "ENABLE:enable" + input_stream: "LANDMARKS:filtered_world_visibility" + output_stream: "FILTERED_LANDMARKS:filtered_world_landmarks" + options: { + [mediapipe.SwitchContainerOptions.ext] { + enable: true + contained_node: { + calculator: "LandmarksSmoothingCalculator" + options: { + [mediapipe.LandmarksSmoothingCalculatorOptions.ext] { + no_filter: {} + } + } + } + contained_node: { + calculator: "LandmarksSmoothingCalculator" + options: { + [mediapipe.LandmarksSmoothingCalculatorOptions.ext] { + one_euro_filter { + # Min cutoff 0.1 results into ~ 0.02 alpha in landmark EMA filter + # when landmark is static. + min_cutoff: 0.1 + # Beta 40.0 in combintation with min_cutoff 0.1 results into ~0.8 + # alpha in landmark EMA filter when landmark is moving fast. + beta: 40.0 + # Derivative cutoff 1.0 results into ~0.17 alpha in landmark + # velocity EMA filter. + derivate_cutoff: 1.0 + # As world landmdarks are predicted in real world 3D coordintates + # in meters (rather than in pixels of input image) prediction + # scale does not depend on the pose size in the image. + disable_value_scaling: true + } + } + } + } + } + } +} + +# Smoothes auxiliary landmark visibilities to reduce jitter. node { calculator: "VisibilitySmoothingCalculator" input_stream: "NORM_LANDMARKS:aux_landmarks" diff --git a/mediapipe/modules/pose_landmark/pose_landmark_full.tflite b/mediapipe/modules/pose_landmark/pose_landmark_full.tflite index 922be204..0f2534cf 100755 Binary files a/mediapipe/modules/pose_landmark/pose_landmark_full.tflite and b/mediapipe/modules/pose_landmark/pose_landmark_full.tflite differ diff --git a/mediapipe/modules/pose_landmark/pose_landmark_gpu.pbtxt b/mediapipe/modules/pose_landmark/pose_landmark_gpu.pbtxt index c4397376..ce053b76 100644 --- a/mediapipe/modules/pose_landmark/pose_landmark_gpu.pbtxt +++ b/mediapipe/modules/pose_landmark/pose_landmark_gpu.pbtxt @@ -37,7 +37,7 @@ input_side_packet: "SMOOTH_LANDMARKS:smooth_landmarks" # functions as set to 1. (int) input_side_packet: "MODEL_COMPLEXITY:model_complexity" -# Pose landmarks within the given ROI. (NormalizedLandmarkList) +# Pose landmarks. (NormalizedLandmarkList) # We have 33 landmarks (see pose_landmark_topology.svg), and there are other # auxiliary key points. # 0 - nose @@ -80,6 +80,14 @@ input_side_packet: "MODEL_COMPLEXITY:model_complexity" # the absence of this packet so that they don't wait for it unnecessarily. output_stream: "LANDMARKS:pose_landmarks" +# Pose world landmarks. (LandmarkList) +# World landmarks are real-world 3D coordinates in meters with the origin at the +# center between hips. WORLD_LANDMARKS shares the same landmark topology as +# LANDMARKS. However, LANDMARKS provides coordinates (in pixels) of a 3D object +# projected onto the 2D image surface, while WORLD_LANDMARKS provides +# coordinates (in meters) of the 3D object itself. +output_stream: "WORLD_LANDMARKS:pose_world_landmarks" + # Extra outputs (for debugging, for instance). # Detected poses. (Detection) output_stream: "DETECTION:pose_detection" @@ -185,6 +193,7 @@ node { input_stream: "ROI:pose_rect" output_stream: "LANDMARKS:unfiltered_pose_landmarks" output_stream: "AUXILIARY_LANDMARKS:unfiltered_auxiliary_landmarks" + output_stream: "WORLD_LANDMARKS:unfiltered_world_landmarks" } # Smoothes landmarks to reduce jitter. @@ -194,8 +203,10 @@ node { input_stream: "IMAGE_SIZE:image_size" input_stream: "NORM_LANDMARKS:unfiltered_pose_landmarks" input_stream: "AUX_NORM_LANDMARKS:unfiltered_auxiliary_landmarks" + input_stream: "WORLD_LANDMARKS:unfiltered_world_landmarks" output_stream: "FILTERED_NORM_LANDMARKS:pose_landmarks" output_stream: "FILTERED_AUX_NORM_LANDMARKS:auxiliary_landmarks" + output_stream: "FILTERED_WORLD_LANDMARKS:pose_world_landmarks" } # Calculates region of interest based on the auxiliary landmarks, to be used in diff --git a/mediapipe/modules/pose_landmark/pose_landmark_heavy.tflite b/mediapipe/modules/pose_landmark/pose_landmark_heavy.tflite index e72fc036..9249c48c 100755 Binary files a/mediapipe/modules/pose_landmark/pose_landmark_heavy.tflite and b/mediapipe/modules/pose_landmark/pose_landmark_heavy.tflite differ diff --git a/mediapipe/modules/pose_landmark/pose_landmark_lite.tflite b/mediapipe/modules/pose_landmark/pose_landmark_lite.tflite index 54e217be..a812fb09 100755 Binary files a/mediapipe/modules/pose_landmark/pose_landmark_lite.tflite and b/mediapipe/modules/pose_landmark/pose_landmark_lite.tflite differ diff --git a/mediapipe/python/BUILD b/mediapipe/python/BUILD index 11fe4583..42ce07f6 100644 --- a/mediapipe/python/BUILD +++ b/mediapipe/python/BUILD @@ -59,11 +59,12 @@ cc_library( "//mediapipe/calculators/core:gate_calculator", "//mediapipe/calculators/core:pass_through_calculator", "//mediapipe/calculators/core:side_packet_to_stream_calculator", - "//mediapipe/calculators/core:split_normalized_landmark_list_calculator", + "//mediapipe/calculators/core:split_landmarks_calculator", "//mediapipe/calculators/core:string_to_int_calculator", "//mediapipe/calculators/image:image_transformation_calculator", "//mediapipe/calculators/util:detection_unique_id_calculator", - "//mediapipe/modules/face_detection:face_detection_front_cpu", + "//mediapipe/modules/face_detection:face_detection_full_range_cpu", + "//mediapipe/modules/face_detection:face_detection_short_range_cpu", "//mediapipe/modules/face_landmark:face_landmark_front_cpu", "//mediapipe/modules/hand_landmark:hand_landmark_tracking_cpu", "//mediapipe/modules/holistic_landmark:holistic_landmark_cpu", diff --git a/mediapipe/python/pybind/calculator_graph.cc b/mediapipe/python/pybind/calculator_graph.cc index a13f03b0..017e16b3 100644 --- a/mediapipe/python/pybind/calculator_graph.cc +++ b/mediapipe/python/pybind/calculator_graph.cc @@ -399,7 +399,12 @@ void CalculatorGraphSubmodule(pybind11::module* module) { stream_name, [callback_fn, stream_name](const Packet& packet) { absl::MutexLock lock(&callback_mutex); - callback_fn(stream_name, packet); + py::gil_scoped_release gil_release; + { + // Acquires GIL before calling Python callback. + py::gil_scoped_acquire gil_acquire; + callback_fn(stream_name, packet); + } return absl::OkStatus(); }, observe_timestamp_bounds)); diff --git a/mediapipe/python/pybind/packet_getter.cc b/mediapipe/python/pybind/packet_getter.cc index 27118440..0dab24ea 100644 --- a/mediapipe/python/pybind/packet_getter.cc +++ b/mediapipe/python/pybind/packet_getter.cc @@ -397,7 +397,7 @@ void InternalPacketGetters(pybind11::module* m) { "_get_serialized_proto", [](const Packet& packet) { // By default, py::bytes is an extra copy of the original std::string - // object: https://github.com/pybind/pybind11/issues/1236 Howeover, when + // object: https://github.com/pybind/pybind11/issues/1236 However, when // Pybind11 performs the C++ to Python transition, it only increases the // py::bytes object's ref count. See the implmentation at line 1583 in // "pybind11/cast.h". diff --git a/mediapipe/python/solutions/drawing_utils.py b/mediapipe/python/solutions/drawing_utils.py index 2e0fd997..7939a445 100644 --- a/mediapipe/python/solutions/drawing_utils.py +++ b/mediapipe/python/solutions/drawing_utils.py @@ -19,6 +19,7 @@ from typing import List, Optional, Tuple, Union import cv2 import dataclasses +import matplotlib.pyplot as plt import numpy as np from mediapipe.framework.formats import detection_pb2 @@ -27,6 +28,7 @@ from mediapipe.framework.formats import landmark_pb2 PRESENCE_THRESHOLD = 0.5 RGB_CHANNELS = 3 +BLACK_COLOR = (0, 0, 0) RED_COLOR = (0, 0, 255) GREEN_COLOR = (0, 128, 0) BLUE_COLOR = (255, 0, 0) @@ -225,3 +227,71 @@ def draw_axis( axis_drawing_spec.thickness) cv2.arrowedLine(image, origin, z_axis, BLUE_COLOR, axis_drawing_spec.thickness) + + +def _normalize_color(color): + return tuple(v / 255. for v in color) + + +def plot_landmarks(landmark_list: landmark_pb2.NormalizedLandmarkList, + connections: Optional[List[Tuple[int, int]]] = None, + landmark_drawing_spec: DrawingSpec = DrawingSpec( + color=RED_COLOR, thickness=5), + connection_drawing_spec: DrawingSpec = DrawingSpec( + color=BLACK_COLOR, thickness=5), + elevation: int = 10, + azimuth: int = 10): + """Plot the landmarks and the connections in matplotlib 3d. + + Args: + landmark_list: A normalized landmark list proto message to be plotted. + connections: A list of landmark index tuples that specifies how landmarks to + be connected. + landmark_drawing_spec: A DrawingSpec object that specifies the landmarks' + drawing settings such as color and line thickness. + connection_drawing_spec: A DrawingSpec object that specifies the + connections' drawing settings such as color and line thickness. + elevation: The elevation from which to view the plot. + azimuth: the azimuth angle to rotate the plot. + Raises: + ValueError: If any connetions contain invalid landmark index. + """ + if not landmark_list: + return + plt.figure(figsize=(10, 10)) + ax = plt.axes(projection='3d') + ax.view_init(elev=elevation, azim=azimuth) + plotted_landmarks = {} + for idx, landmark in enumerate(landmark_list.landmark): + if ((landmark.HasField('visibility') and + landmark.visibility < VISIBILITY_THRESHOLD) or + (landmark.HasField('presence') and + landmark.presence < PRESENCE_THRESHOLD)): + continue + ax.scatter3D( + xs=[-landmark.z], + ys=[landmark.x], + zs=[-landmark.y], + color=_normalize_color(landmark_drawing_spec.color[::-1]), + linewidth=landmark_drawing_spec.thickness) + plotted_landmarks[idx] = (-landmark.z, landmark.x, -landmark.y) + if connections: + num_landmarks = len(landmark_list.landmark) + # Draws the connections if the start and end landmarks are both visible. + for connection in connections: + start_idx = connection[0] + end_idx = connection[1] + if not (0 <= start_idx < num_landmarks and 0 <= end_idx < num_landmarks): + raise ValueError(f'Landmark index is out of range. Invalid connection ' + f'from landmark #{start_idx} to landmark #{end_idx}.') + if start_idx in plotted_landmarks and end_idx in plotted_landmarks: + landmark_pair = [ + plotted_landmarks[start_idx], plotted_landmarks[end_idx] + ] + ax.plot3D( + xs=[landmark_pair[0][0], landmark_pair[1][0]], + ys=[landmark_pair[0][1], landmark_pair[1][1]], + zs=[landmark_pair[0][2], landmark_pair[1][2]], + color=_normalize_color(connection_drawing_spec.color[::-1]), + linewidth=connection_drawing_spec.thickness) + plt.show() diff --git a/mediapipe/python/solutions/face_detection.py b/mediapipe/python/solutions/face_detection.py index ef5eefdb..6f62ae92 100644 --- a/mediapipe/python/solutions/face_detection.py +++ b/mediapipe/python/solutions/face_detection.py @@ -28,7 +28,8 @@ from mediapipe.calculators.util import non_max_suppression_calculator_pb2 # pylint: enable=unused-import from mediapipe.python.solution_base import SolutionBase -BINARYPB_FILE_PATH = 'mediapipe/modules/face_detection/face_detection_front_cpu.binarypb' +SHORT_RANGE_GRAPH_FILE_PATH = 'mediapipe/modules/face_detection/face_detection_short_range_cpu.binarypb' +FULL_RANGE_GRAPH_FILE_PATH = 'mediapipe/modules/face_detection/face_detection_full_range_cpu.binarypb' def get_key_point( @@ -69,18 +70,26 @@ class FaceDetection(SolutionBase): for usage examples. """ - def __init__(self, min_detection_confidence=0.5): + def __init__(self, min_detection_confidence=0.5, model_selection=0): """Initializes a MediaPipe Face Detection object. Args: min_detection_confidence: Minimum confidence value ([0.0, 1.0]) for face detection to be considered successful. See details in https://solutions.mediapipe.dev/face_detection#min_detection_confidence. + model_selection: 0 or 1. 0 to select a short-range model that works + best for faces within 2 meters from the camera, and 1 for a full-range + model best for faces within 5 meters. See details in + https://solutions.mediapipe.dev/face_detection#model_selection. """ + + binary_graph_path = FULL_RANGE_GRAPH_FILE_PATH if model_selection == 1 else SHORT_RANGE_GRAPH_FILE_PATH + subgraph_name = 'facedetectionfullrangecommon' if model_selection == 1 else 'facedetectionshortrangecommon' + super().__init__( - binary_graph_path=BINARYPB_FILE_PATH, + binary_graph_path=binary_graph_path, calculator_params={ - 'facedetectionfrontcommon__TensorsToDetectionsCalculator.min_score_thresh': + subgraph_name + '__TensorsToDetectionsCalculator.min_score_thresh': min_detection_confidence, }, outputs=['detections']) diff --git a/mediapipe/python/solutions/face_detection_test.py b/mediapipe/python/solutions/face_detection_test.py index f4185ea4..314079ac 100644 --- a/mediapipe/python/solutions/face_detection_test.py +++ b/mediapipe/python/solutions/face_detection_test.py @@ -18,6 +18,7 @@ import tempfile # pylint: disable=unused-import from typing import NamedTuple from absl.testing import absltest +from absl.testing import parameterized import cv2 import numpy as np import numpy.testing as npt @@ -28,12 +29,14 @@ from mediapipe.python.solutions import drawing_utils as mp_drawing from mediapipe.python.solutions import face_detection as mp_faces TEST_IMAGE_PATH = 'mediapipe/python/solutions/testdata' -EXPECTED_FACE_KEY_POINTS = [[182, 363], [186, 460], [241, 420], [284, 417], - [199, 295], [198, 502]] +SHORT_RANGE_EXPECTED_FACE_KEY_POINTS = [[363, 182], [460, 186], [420, 241], + [417, 284], [295, 199], [502, 198]] +FULL_RANGE_EXPECTED_FACE_KEY_POINTS = [[363, 181], [455, 181], [413, 233], + [411, 278], [306, 204], [499, 207]] DIFF_THRESHOLD = 5 # pixels -class FaceDetectionTest(absltest.TestCase): +class FaceDetectionTest(parameterized.TestCase): def _annotate(self, frame: np.ndarray, results: NamedTuple, idx: int): for detection in results.detections: @@ -55,20 +58,30 @@ class FaceDetectionTest(absltest.TestCase): results = faces.process(image) self.assertIsNone(results.detections) - def test_face(self): + @parameterized.named_parameters(('short_range_model', 0), + ('full_range_model', 1)) + def test_face(self, model_selection): image_path = os.path.join(os.path.dirname(__file__), 'testdata/portrait.jpg') image = cv2.imread(image_path) - with mp_faces.FaceDetection(min_detection_confidence=0.5) as faces: + rows, cols, _ = image.shape + with mp_faces.FaceDetection( + min_detection_confidence=0.5, model_selection=model_selection) as faces: for idx in range(5): results = faces.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB)) self._annotate(image.copy(), results, idx) location_data = results.detections[0].location_data - x = [keypoint.x for keypoint in location_data.relative_keypoints] - y = [keypoint.y for keypoint in location_data.relative_keypoints] - face_keypoints = np.transpose(np.stack((y, x))) * image.shape[0:2] - prediction_error = np.abs( - np.asarray(face_keypoints) - np.asarray(EXPECTED_FACE_KEY_POINTS)) + x = [keypoint.x * cols for keypoint in location_data.relative_keypoints] + y = [keypoint.y * rows for keypoint in location_data.relative_keypoints] + face_keypoints = np.column_stack((x, y)) + if model_selection == 0: + prediction_error = np.abs( + np.asarray(face_keypoints) - + np.asarray(SHORT_RANGE_EXPECTED_FACE_KEY_POINTS)) + else: + prediction_error = np.abs( + np.asarray(face_keypoints) - + np.asarray(FULL_RANGE_EXPECTED_FACE_KEY_POINTS)) self.assertLen(results.detections, 1) self.assertLen(location_data.relative_keypoints, 6) diff --git a/mediapipe/python/solutions/face_mesh.py b/mediapipe/python/solutions/face_mesh.py index 85b7efdc..670a3817 100644 --- a/mediapipe/python/solutions/face_mesh.py +++ b/mediapipe/python/solutions/face_mesh.py @@ -213,7 +213,7 @@ class FaceMesh(SolutionBase): .ConstantSidePacketCalculatorOptions.ConstantSidePacket( bool_value=not static_image_mode) ], - 'facedetectionfrontcpu__TensorsToDetectionsCalculator.min_score_thresh': + 'facedetectionshortrangecpu__TensorsToDetectionsCalculator.min_score_thresh': min_detection_confidence, 'facelandmarkcpu__ThresholdingCalculator.threshold': min_tracking_confidence, diff --git a/mediapipe/python/solutions/face_mesh_test.py b/mediapipe/python/solutions/face_mesh_test.py index 2d850387..9e611e5b 100644 --- a/mediapipe/python/solutions/face_mesh_test.py +++ b/mediapipe/python/solutions/face_mesh_test.py @@ -32,38 +32,38 @@ from mediapipe.python.solutions import face_mesh as mp_faces TEST_IMAGE_PATH = 'mediapipe/python/solutions/testdata' DIFF_THRESHOLD = 5 # pixels EYE_INDICES_TO_LANDMARKS = { - 33: [178, 345], - 7: [179, 348], - 163: [178, 352], - 144: [179, 357], - 145: [179, 365], - 153: [179, 371], - 154: [178, 378], - 155: [177, 381], - 133: [177, 383], - 246: [175, 347], - 161: [174, 350], - 160: [172, 355], - 159: [170, 362], - 158: [171, 368], - 157: [172, 375], - 173: [175, 380], - 263: [176, 467], - 249: [177, 464], - 390: [177, 460], - 373: [178, 455], - 374: [179, 448], - 380: [179, 441], - 381: [178, 435], - 382: [177, 432], - 362: [177, 430], - 466: [175, 465], - 388: [173, 462], - 387: [171, 457], - 386: [170, 450], - 385: [171, 444], - 384: [172, 437], - 398: [175, 432] + 33: [345, 178], + 7: [348, 179], + 163: [352, 178], + 144: [357, 179], + 145: [365, 179], + 153: [371, 179], + 154: [378, 178], + 155: [381, 177], + 133: [383, 177], + 246: [347, 175], + 161: [350, 174], + 160: [355, 172], + 159: [362, 170], + 158: [368, 171], + 157: [375, 172], + 173: [380, 175], + 263: [467, 176], + 249: [464, 177], + 390: [460, 177], + 373: [455, 178], + 374: [448, 179], + 380: [441, 179], + 381: [435, 178], + 382: [432, 177], + 362: [430, 177], + 466: [465, 175], + 388: [462, 173], + 387: [457, 171], + 386: [450, 170], + 385: [444, 171], + 384: [437, 172], + 398: [432, 175] } @@ -99,6 +99,7 @@ class FaceMeshTest(parameterized.TestCase): image_path = os.path.join(os.path.dirname(__file__), 'testdata/portrait.jpg') image = cv2.imread(image_path) + rows, cols, _ = image.shape with mp_faces.FaceMesh( static_image_mode=static_image_mode, min_detection_confidence=0.5) as faces: @@ -108,9 +109,9 @@ class FaceMeshTest(parameterized.TestCase): multi_face_landmarks = [] for landmarks in results.multi_face_landmarks: self.assertLen(landmarks.landmark, 468) - x = [landmark.x for landmark in landmarks.landmark] - y = [landmark.y for landmark in landmarks.landmark] - face_landmarks = np.transpose(np.stack((y, x))) * image.shape[0:2] + x = [landmark.x * cols for landmark in landmarks.landmark] + y = [landmark.y * rows for landmark in landmarks.landmark] + face_landmarks = np.column_stack((x, y)) multi_face_landmarks.append(face_landmarks) self.assertLen(multi_face_landmarks, 1) # Verify the eye landmarks are correct as sanity check. diff --git a/mediapipe/python/solutions/hands_test.py b/mediapipe/python/solutions/hands_test.py index 7e262c1e..2bd9e254 100644 --- a/mediapipe/python/solutions/hands_test.py +++ b/mediapipe/python/solutions/hands_test.py @@ -31,20 +31,20 @@ from mediapipe.python.solutions import hands as mp_hands TEST_IMAGE_PATH = 'mediapipe/python/solutions/testdata' DIFF_THRESHOLD = 15 # pixels -EXPECTED_HAND_COORDINATES_PREDICTION = [[[345, 144], [323, 211], [286, 257], - [237, 289], [203, 322], [216, 219], - [138, 238], [90, 249], [51, 253], - [204, 177], [115, 184], [60, 187], - [19, 185], [208, 138], [127, 131], - [77, 124], [36, 117], [222, 106], - [159, 92], [124, 79], [93, 68]], - [[40, 577], [56, 504], [94, 459], - [146, 429], [182, 397], [167, 496], - [245, 479], [292, 469], [330, 464], - [177, 540], [265, 534], [319, 533], - [360, 536], [172, 581], [252, 587], - [304, 593], [346, 599], [157, 615], - [223, 628], [258, 638], [288, 648]]] +EXPECTED_HAND_COORDINATES_PREDICTION = [[[144, 345], [211, 323], [257, 286], + [289, 237], [322, 203], [219, 216], + [238, 138], [249, 90], [253, 51], + [177, 204], [184, 115], [187, 60], + [185, 19], [138, 208], [131, 127], + [124, 77], [117, 36], [106, 222], + [92, 159], [79, 124], [68, 93]], + [[577, 40], [504, 56], [459, 94], + [429, 146], [397, 182], [496, 167], + [479, 245], [469, 292], [464, 330], + [540, 177], [534, 265], [533, 319], + [536, 360], [581, 172], [587, 252], + [593, 304], [599, 346], [615, 157], + [628, 223], [638, 258], [648, 288]]] class HandsTest(parameterized.TestCase): @@ -88,11 +88,12 @@ class HandsTest(parameterized.TestCase): for handedness in results.multi_handedness ] multi_hand_coordinates = [] + rows, cols, _ = image.shape for landmarks in results.multi_hand_landmarks: self.assertLen(landmarks.landmark, 21) - x = [landmark.x for landmark in landmarks.landmark] - y = [landmark.y for landmark in landmarks.landmark] - hand_coordinates = np.transpose(np.stack((y, x))) * image.shape[0:2] + x = [landmark.x * cols for landmark in landmarks.landmark] + y = [landmark.y * rows for landmark in landmarks.landmark] + hand_coordinates = np.column_stack((x, y)) multi_hand_coordinates.append(hand_coordinates) self.assertLen(handedness, 2) self.assertLen(multi_hand_coordinates, 2) diff --git a/mediapipe/python/solutions/holistic.py b/mediapipe/python/solutions/holistic.py index 64b63ab4..53d28174 100644 --- a/mediapipe/python/solutions/holistic.py +++ b/mediapipe/python/solutions/holistic.py @@ -116,8 +116,8 @@ class Holistic(SolutionBase): min_tracking_confidence, }, outputs=[ - 'pose_landmarks', 'left_hand_landmarks', 'right_hand_landmarks', - 'face_landmarks' + 'pose_landmarks', 'pose_world_landmarks', 'left_hand_landmarks', + 'right_hand_landmarks', 'face_landmarks' ]) def process(self, image: np.ndarray) -> NamedTuple: @@ -131,17 +131,22 @@ class Holistic(SolutionBase): ValueError: If the input image is not three channel RGB. Returns: - A NamedTuple that has four fields: - 1) "pose_landmarks" field that contains the pose landmarks on the most - prominent person detected. - 2) "left_hand_landmarks" and "right_hand_landmarks" fields that contain - the left and right hand landmarks of the most prominent person detected. - 3) "face_landmarks" field that contains the face landmarks of the most - prominent person detected. + A NamedTuple that has five fields describing the landmarks on the most + prominate person detected: + 1) "pose_landmarks" field that contains the pose landmarks. + 2) "pose_world_landmarks" field that contains the pose landmarks in + real-world 3D coordinates that are in meters with the origin at the + center between hips. + 3) "left_hand_landmarks" field that contains the left-hand landmarks. + 4) "right_hand_landmarks" field that contains the right-hand landmarks. + 5) "face_landmarks" field that contains the face landmarks. """ results = super().process(input_data={'image': image}) if results.pose_landmarks: for landmark in results.pose_landmarks.landmark: landmark.ClearField('presence') + if results.pose_world_landmarks: + for landmark in results.pose_world_landmarks.landmark: + landmark.ClearField('presence') return results diff --git a/mediapipe/python/solutions/pose.py b/mediapipe/python/solutions/pose.py index e25fe626..7e0ec8b4 100644 --- a/mediapipe/python/solutions/pose.py +++ b/mediapipe/python/solutions/pose.py @@ -185,7 +185,7 @@ class Pose(SolutionBase): 'poselandmarkcpu__poselandmarkbyroicpu__ThresholdingCalculator.threshold': min_tracking_confidence, }, - outputs=['pose_landmarks']) + outputs=['pose_landmarks', 'pose_world_landmarks']) def process(self, image: np.ndarray) -> NamedTuple: """Processes an RGB image and returns the pose landmarks on the most prominent person detected. @@ -198,12 +198,19 @@ class Pose(SolutionBase): ValueError: If the input image is not three channel RGB. Returns: - A NamedTuple object with a "pose_landmarks" field that contains the pose - landmarks on the most prominent person detected. + A NamedTuple that has two fields describing the landmarks on the most + prominate person detected: + 1) "pose_landmarks" field that contains the pose landmarks. + 2) "pose_world_landmarks" field that contains the pose landmarks in + real-world 3D coordinates that are in meters with the origin at the + center between hips. """ results = super().process(input_data={'image': image}) if results.pose_landmarks: for landmark in results.pose_landmarks.landmark: landmark.ClearField('presence') + if results.pose_world_landmarks: + for landmark in results.pose_world_landmarks.landmark: + landmark.ClearField('presence') return results diff --git a/mediapipe/python/solutions/pose_test.py b/mediapipe/python/solutions/pose_test.py index 2fb19991..31c5df16 100644 --- a/mediapipe/python/solutions/pose_test.py +++ b/mediapipe/python/solutions/pose_test.py @@ -42,6 +42,20 @@ EXPECTED_POSE_LANDMARKS = np.array([[460, 283], [467, 273], [471, 273], [467, 471], [612, 550], [358, 490], [701, 613], [349, 611], [709, 624], [363, 630], [730, 633], [303, 628]]) +WORLD_DIFF_THRESHOLD = 0.2 # meters +EXPECTED_POSE_WORLD_LANDMARKS = np.array([ + [-0.11, -0.59, -0.15], [-0.09, -0.64, -0.16], [-0.09, -0.64, -0.16], + [-0.09, -0.64, -0.16], [-0.11, -0.64, -0.14], [-0.11, -0.64, -0.14], + [-0.11, -0.64, -0.14], [0.01, -0.65, -0.15], [-0.06, -0.64, -0.05], + [-0.07, -0.57, -0.15], [-0.09, -0.57, -0.12], [0.18, -0.49, -0.09], + [-0.14, -0.5, -0.03], [0.41, -0.48, -0.11], [-0.42, -0.5, -0.02], + [0.64, -0.49, -0.17], [-0.63, -0.51, -0.13], [0.7, -0.5, -0.19], + [-0.71, -0.53, -0.15], [0.72, -0.51, -0.23], [-0.69, -0.54, -0.19], + [0.66, -0.49, -0.19], [-0.64, -0.52, -0.15], [0.09, 0., -0.04], + [-0.09, -0., 0.03], [0.41, 0.23, -0.09], [-0.43, 0.1, -0.11], + [0.69, 0.49, -0.04], [-0.48, 0.47, -0.02], [0.72, 0.52, -0.04], + [-0.48, 0.51, -0.02], [0.8, 0.5, -0.14], [-0.59, 0.52, -0.11], +]) class PoseTest(parameterized.TestCase): @@ -51,6 +65,10 @@ class PoseTest(parameterized.TestCase): return np.asarray([(lmk.x * cols, lmk.y * rows, lmk.z * cols) for lmk in landmark_list.landmark]) + def _world_landmarks_list_to_array(self, landmark_list): + return np.asarray([(lmk.x, lmk.y, lmk.z) + for lmk in landmark_list.landmark]) + def _assert_diff_less(self, array1, array2, threshold): npt.assert_array_less(np.abs(array1 - array2), threshold) @@ -87,11 +105,15 @@ class PoseTest(parameterized.TestCase): model_complexity=model_complexity) as pose: for idx in range(num_frames): results = pose.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB)) + # TODO: Add rendering of world 3D when supported. self._annotate(image.copy(), results, idx) self._assert_diff_less( self._landmarks_list_to_array(results.pose_landmarks, image.shape)[:, :2], EXPECTED_POSE_LANDMARKS, DIFF_THRESHOLD) + self._assert_diff_less( + self._world_landmarks_list_to_array(results.pose_world_landmarks), + EXPECTED_POSE_WORLD_LANDMARKS, WORLD_DIFF_THRESHOLD) @parameterized.named_parameters( ('full', 1, 'pose_squats.full.npz')) @@ -99,9 +121,9 @@ class PoseTest(parameterized.TestCase): """Tests pose models on a video.""" # If set to `True` will dump actual predictions to .npz and JSON files. dump_predictions = False - # Set threshold for comparing actual and expected predictions in pixels. - diff_threshold = 50 + diff_threshold = 15 + world_diff_threshold = 0.1 video_path = os.path.join(os.path.dirname(__file__), 'testdata/pose_squats.mp4') @@ -111,6 +133,7 @@ class PoseTest(parameterized.TestCase): # Predict pose landmarks for each frame. video_cap = cv2.VideoCapture(video_path) actual_per_frame = [] + actual_world_per_frame = [] frame_idx = 0 with mp_pose.Pose(static_image_mode=False, model_complexity=model_complexity) as pose: @@ -125,28 +148,35 @@ class PoseTest(parameterized.TestCase): result = pose.process(image=input_frame) pose_landmarks = self._landmarks_list_to_array(result.pose_landmarks, input_frame.shape) + pose_world_landmarks = self._world_landmarks_list_to_array( + result.pose_world_landmarks) actual_per_frame.append(pose_landmarks) + actual_world_per_frame.append(pose_world_landmarks) input_frame = cv2.cvtColor(input_frame, cv2.COLOR_RGB2BGR) self._annotate(input_frame, result, frame_idx) frame_idx += 1 - actual = np.asarray(actual_per_frame) + actual = np.array(actual_per_frame) + actual_world = np.array(actual_world_per_frame) if dump_predictions: # Dump .npz with tempfile.NamedTemporaryFile(delete=False) as tmp_file: - np.savez(tmp_file, predictions=np.array(actual)) + np.savez(tmp_file, predictions=actual, predictions_world=actual_world) print('Predictions saved as .npz to {}'.format(tmp_file.name)) # Dump JSON with tempfile.NamedTemporaryFile(delete=False) as tmp_file: with open(tmp_file.name, 'w') as fl: - dump_data = {'predictions': np.around(actual, 3).tolist()} + dump_data = { + 'predictions': np.around(actual, 3).tolist(), + 'predictions_world': np.around(actual_world, 3).tolist() + } fl.write(json.dumps(dump_data, indent=2, separators=(',', ': '))) print('Predictions saved as JSON to {}'.format(tmp_file.name)) - # Validate actual vs. expected predictions. + # Validate actual vs. expected landmarks. expected = np.load(expected_path)['predictions'] assert actual.shape == expected.shape, ( 'Unexpected shape of predictions: {} instead of {}'.format( @@ -154,6 +184,14 @@ class PoseTest(parameterized.TestCase): self._assert_diff_less( actual[..., :2], expected[..., :2], threshold=diff_threshold) + # Validate actual vs. expected world landmarks. + expected_world = np.load(expected_path)['predictions_world'] + assert actual_world.shape == expected_world.shape, ( + 'Unexpected shape of world predictions: {} instead of {}'.format( + actual_world.shape, expected_world.shape)) + self._assert_diff_less( + actual_world, expected_world, threshold=world_diff_threshold) + if __name__ == '__main__': absltest.main() diff --git a/mediapipe/util/BUILD b/mediapipe/util/BUILD index da122442..11f08ade 100644 --- a/mediapipe/util/BUILD +++ b/mediapipe/util/BUILD @@ -176,6 +176,7 @@ cc_library( deps = [ ":resource_util_custom", "@com_google_absl//absl/container:flat_hash_map", + "//mediapipe/framework/deps:file_path", "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:singleton", "//mediapipe/framework/port:status", @@ -184,7 +185,6 @@ cc_library( "@com_google_absl//absl/strings", ] + select({ "//conditions:default": [ - "//mediapipe/framework/deps:file_path", "@com_google_absl//absl/flags:flag", ], "//mediapipe:android": [ @@ -193,7 +193,6 @@ cc_library( ], "//mediapipe:ios": [], "//mediapipe:macos": [ - "//mediapipe/framework/deps:file_path", "@com_google_absl//absl/flags:flag", ], }), @@ -295,3 +294,29 @@ cc_test( "@eigen_archive//:eigen3", ], ) + +cc_library( + name = "packet_test_util", + testonly = 1, + hdrs = ["packet_test_util.h"], + visibility = ["//visibility:public"], + deps = [ + "//mediapipe/framework:demangle", + "//mediapipe/framework:packet", + "//mediapipe/framework:timestamp", + "//mediapipe/framework/port:gtest_main", + ], +) + +cc_test( + name = "packet_test_util_test", + size = "small", + srcs = ["packet_test_util_test.cc"], + deps = [ + ":packet_test_util", + "//mediapipe/framework:packet", + "//mediapipe/framework:port", + "//mediapipe/framework:timestamp", + "//mediapipe/framework/port:gtest_main", + ], +) diff --git a/mediapipe/util/packet_test_util.h b/mediapipe/util/packet_test_util.h new file mode 100644 index 00000000..106d7f8d --- /dev/null +++ b/mediapipe/util/packet_test_util.h @@ -0,0 +1,123 @@ +// Copyright 2021 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Utilities that help to make assertions about packet contents in tests. + +#ifndef MEDIAPIPE_UTIL_PACKET_TEST_UTIL_H_ +#define MEDIAPIPE_UTIL_PACKET_TEST_UTIL_H_ + +#include +#include +#include + +#include "mediapipe/framework/demangle.h" +#include "mediapipe/framework/packet.h" +#include "mediapipe/framework/port/gmock.h" +#include "mediapipe/framework/port/gtest.h" +#include "mediapipe/framework/timestamp.h" + +namespace mediapipe { + +namespace internal { + +template +class PacketMatcher : public ::testing::MatcherInterface { + public: + template + explicit PacketMatcher(InnerMatcher inner_matcher) + : inner_matcher_( + ::testing::SafeMatcherCast(inner_matcher)) {} + + // Returns true iff the packet contains value of PayloadType satisfying + // the inner matcher. + bool MatchAndExplain( + const Packet& packet, + ::testing::MatchResultListener* listener) const override { + if (!packet.ValidateAsType().ok()) { + *listener << packet.DebugString() << " does not contain expected type " + << ExpectedTypeName(); + return false; + } + ::testing::StringMatchResultListener match_listener; + const PayloadType& payload = packet.Get(); + const bool matches = + inner_matcher_.MatchAndExplain(payload, &match_listener); + const std::string explanation = match_listener.str(); + *listener << packet.DebugString() << " containing value " + << ::testing::PrintToString(payload); + if (!explanation.empty()) { + *listener << ", which " << explanation; + } + return matches; + } + + void DescribeTo(std::ostream* os) const override { + *os << "packet contains value of type " << ExpectedTypeName() << " that "; + inner_matcher_.DescribeTo(os); + } + + void DescribeNegationTo(std::ostream* os) const override { + *os << "packet does not contain value of type " << ExpectedTypeName() + << " that "; + inner_matcher_.DescribeNegationTo(os); + } + + private: + static std::string ExpectedTypeName() { + return ::mediapipe::Demangle(typeid(PayloadType).name()); + } + + const ::testing::Matcher inner_matcher_; +}; + +} // namespace internal + +// Creates matcher validating that the packet contains value of expected type +// and satisfying the provided inner matcher. +// +// PayloadType template parameter has to be specified explicitly, but matcher +// type can be inferred. Example: +// +// EXPECT_THAT(MakePacket(42), PacketContains(Eq(42))) +template +inline ::testing::Matcher PacketContains( + InnerMatcher inner_matcher) { + return ::testing::MakeMatcher( + new internal::PacketMatcher(inner_matcher)); +} + +// Creates matcher validating the packet's timestamp satisfies the provided +// timestamp_matcher. It also checks that the packet contains value of expected +// type and satisfies the provided content matcher. +// +// PayloadType template parameter has to be specified explicitly, but matcher +// type can be inferred. Example: +// +// EXPECT_THAT(MakePacket(42).At(Timestamp(20)), +// PacketContainsTimestampAndPayload( // +// Eq(Timestamp(20)), +// Eq(42))) +template +inline ::testing::Matcher PacketContainsTimestampAndPayload( + TimestampMatcher timestamp_matcher, ContentMatcher content_matcher) { + return testing::AllOf( + testing::Property("Packet::Timestamp", &Packet::Timestamp, + timestamp_matcher), + PacketContains(content_matcher)); +} + +} // namespace mediapipe + +#endif // MEDIAPIPE_UTIL_PACKET_TEST_UTIL_H_ diff --git a/mediapipe/util/packet_test_util_test.cc b/mediapipe/util/packet_test_util_test.cc new file mode 100644 index 00000000..58340c0c --- /dev/null +++ b/mediapipe/util/packet_test_util_test.cc @@ -0,0 +1,108 @@ +// Copyright 2021 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +#include "mediapipe/util/packet_test_util.h" + +#include + +#include "mediapipe/framework/packet.h" +#include "mediapipe/framework/port.h" +#include "mediapipe/framework/port/gmock.h" +#include "mediapipe/framework/port/gtest-spi.h" +#include "mediapipe/framework/timestamp.h" + +namespace mediapipe { +namespace { + +using ::testing::Eq; + +TEST(PacketTestUtilTest, Matches) { + const Packet int_packet = MakePacket(42); + EXPECT_THAT(int_packet, PacketContains(Eq(42))); +} + +TEST(PacketTestUtilTest, MatchesContentWithMatchingTimestamp) { + const Packet int_packet = MakePacket(42).At(Timestamp::PostStream()); + EXPECT_THAT(int_packet, PacketContainsTimestampAndPayload( + Eq(Timestamp::PostStream()), Eq(42))); +} + +TEST(PacketTestUtilTest, MatchesContentWithMismatchingTimestamp) { + const Packet int_packet = MakePacket(42).At(Timestamp(0ll)); + EXPECT_NONFATAL_FAILURE( + { + EXPECT_THAT(int_packet, PacketContainsTimestampAndPayload( + Eq(Timestamp::PostStream()), Eq(42))); + }, + "`Packet::Timestamp` is equal to Timestamp::PostStream()"); +} + +TEST(PacketTestUtilTest, DoesNotMatch) { + const Packet int_packet = MakePacket(42); + EXPECT_NONFATAL_FAILURE( + { EXPECT_THAT(int_packet, PacketContains(Eq(47))); }, + "containing value 42"); +} + +TEST(PacketTestUtilTest, DoesNotMatchContentWithMatchingTimestamp) { + const Packet int_packet = MakePacket(42).At(Timestamp(0ll)); + EXPECT_NONFATAL_FAILURE( + { + EXPECT_THAT(int_packet, PacketContainsTimestampAndPayload( + Eq(Timestamp(0ll)), Eq(47))); + }, + "type int that is equal to 47"); +} + +TEST(PacketTestUtilTest, DoesNotMatchContentWithMismatchingTimestamp) { + const Packet int_packet = MakePacket(42).At(Timestamp(0ll)); + EXPECT_NONFATAL_FAILURE( + { + EXPECT_THAT(int_packet, PacketContainsTimestampAndPayload( + Eq(Timestamp(20ll)), Eq(47))); + }, + "`Packet::Timestamp` is equal to 20) and (packet contains value of type " + "int that is equal to 47"); +} + +TEST(PacketTestUtilTest, TypeMismatch) { + const Packet string_packet = MakePacket("42"); + EXPECT_NONFATAL_FAILURE( + { EXPECT_THAT(string_packet, PacketContains(Eq(42))); }, + "does not contain expected type int"); +} + +TEST(PacketTestUtilTest, TypeMismatchContentWithMatchingTimestamp) { + const Packet int_packet = MakePacket("42").At(Timestamp(0ll)); + EXPECT_NONFATAL_FAILURE( + { + EXPECT_THAT(int_packet, PacketContainsTimestampAndPayload( + Eq(Timestamp(0ll)), Eq(47))); + }, + "does not contain expected type int"); +} + +TEST(PacketTestUtilTest, TypeMismatchContentWithMismatchingTimestamp) { + const Packet int_packet = MakePacket("42").At(Timestamp(0ll)); + EXPECT_NONFATAL_FAILURE( + { + EXPECT_THAT(int_packet, PacketContainsTimestampAndPayload( + Eq(Timestamp::PreStream()), Eq(47))); + }, + "`Packet::Timestamp` is equal to Timestamp::PreStream()) and (packet " + "contains value of type int that is equal to 47"); +} + +} // namespace +} // namespace mediapipe diff --git a/mediapipe/util/tflite/tflite_model_loader.cc b/mediapipe/util/tflite/tflite_model_loader.cc index abd0e725..aab94ccb 100644 --- a/mediapipe/util/tflite/tflite_model_loader.cc +++ b/mediapipe/util/tflite/tflite_model_loader.cc @@ -29,11 +29,9 @@ absl::StatusOr> TfLiteModelLoader::LoadFromPath( // TODO: get rid of manual resolving with PathToResourceAsFile // as soon as it's incorporated into GetResourceContents. if (!status_or_content.ok()) { - LOG(WARNING) - << "Trying to resolve path manually as GetResourceContents failed: " - << status_or_content.message(); ASSIGN_OR_RETURN(auto resolved_path, mediapipe::PathToResourceAsFile(model_path)); + VLOG(2) << "Loading the model from " << resolved_path; MP_RETURN_IF_ERROR( mediapipe::GetResourceContents(resolved_path, &model_blob)); } diff --git a/requirements.txt b/requirements.txt index fcb9ad31..46f0fdad 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ absl-py attrs>=19.1.0 +matplotlib numpy opencv-contrib-python protobuf>=3.11.4 diff --git a/setup.py b/setup.py index 31e5195b..24fb2e37 100644 --- a/setup.py +++ b/setup.py @@ -222,7 +222,8 @@ class BuildBinaryGraphs(build.build): def run(self): _check_bazel() binary_graphs = [ - 'face_detection/face_detection_front_cpu', + 'face_detection/face_detection_short_range_cpu', + 'face_detection/face_detection_full_range_cpu', 'face_landmark/face_landmark_front_cpu', 'hand_landmark/hand_landmark_tracking_cpu', 'holistic_landmark/holistic_landmark_cpu', 'objectron/objectron_cpu', @@ -241,6 +242,7 @@ class BuildBinaryGraphs(build.build): 'bazel', 'build', '--compilation_mode=opt', + '--copt=-DNDEBUG', '--define=MEDIAPIPE_DISABLE_GPU=1', '--action_env=PYTHON_BIN_PATH=' + _normalize_path(sys.executable), os.path.join('mediapipe/modules/', graph_path), @@ -297,6 +299,7 @@ class BuildBazelExtension(build_ext.build_ext): 'bazel', 'build', '--compilation_mode=opt', + '--copt=-DNDEBUG', '--define=MEDIAPIPE_DISABLE_GPU=1', '--action_env=PYTHON_BIN_PATH=' + _normalize_path(sys.executable), str(ext.bazel_target + '.so'), @@ -416,7 +419,7 @@ setuptools.setup( version=__version__, url='https://github.com/google/mediapipe', description='MediaPipe is the simplest way for researchers and developers to build world-class ML solutions and applications for mobile, edge, cloud and the web.', - author='MediaPipe Authors', + author='The MediaPipe Authors', author_email='mediapipe@google.com', long_description=_get_long_description(), long_description_content_type='text/markdown', diff --git a/third_party/BUILD b/third_party/BUILD index 5800098f..e2044cfd 100644 --- a/third_party/BUILD +++ b/third_party/BUILD @@ -296,10 +296,20 @@ java_plugin( ], ) +java_plugin( + name = "autobuilder_plugin", + processor_class = "com.google.auto.value.processor.AutoBuilderProcessor", + deps = [ + "@maven//:com_google_auto_value_auto_value", + "@maven//:com_google_auto_value_auto_value_annotations", + ], +) + java_library( name = "autovalue", exported_plugins = [ ":autovalue_plugin", + ":autobuilder_plugin", ], neverlink = 1, exports = [