talksik/mediapipe
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Project import generated by Copybara.

Browse Source 
GitOrigin-RevId: 33adfdf31f3a5cbf9edc07ee1ea583e95080bdc5
This commit is contained in:
MediaPipe Team
2021-06-24 17:55:26 -04:00
committed by chuoling
parent b544a314b3
commit 139237092f
151 changed files with 4023 additions and 1001 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mediapipe/modules/face_detection/BUILD
+58 -22
View File
@@ -22,11 +22,11 @@ licenses(["notice"])
package(default_visibility = ["//visibility:public"])
mediapipe_simple_subgraph(
name = "face_detection_front_by_roi_cpu",
graph = "face_detection_front_by_roi_cpu.pbtxt",
register_as = "FaceDetectionFrontByRoiCpu",
name = "face_detection_short_range_by_roi_cpu",
graph = "face_detection_short_range_by_roi_cpu.pbtxt",
register_as = "FaceDetectionShortRangeByRoiCpu",
deps = [
":face_detection_front_common",
":face_detection_short_range_common",
"//mediapipe/calculators/tensor:image_to_tensor_calculator",
"//mediapipe/calculators/tensor:inference_calculator",
"//mediapipe/calculators/util:to_image_calculator",
@@ -34,11 +34,11 @@ mediapipe_simple_subgraph(
)
mediapipe_simple_subgraph(
name = "face_detection_front_by_roi_gpu",
graph = "face_detection_front_by_roi_gpu.pbtxt",
register_as = "FaceDetectionFrontByRoiGpu",
name = "face_detection_short_range_by_roi_gpu",
graph = "face_detection_short_range_by_roi_gpu.pbtxt",
register_as = "FaceDetectionShortRangeByRoiGpu",
deps = [
":face_detection_front_common",
":face_detection_short_range_common",
"//mediapipe/calculators/tensor:image_to_tensor_calculator",
"//mediapipe/calculators/tensor:inference_calculator",
"//mediapipe/calculators/util:to_image_calculator",
@@ -46,11 +46,11 @@ mediapipe_simple_subgraph(
)
mediapipe_simple_subgraph(
name = "face_detection_front_cpu",
graph = "face_detection_front_cpu.pbtxt",
register_as = "FaceDetectionFrontCpu",
name = "face_detection_short_range_cpu",
graph = "face_detection_short_range_cpu.pbtxt",
register_as = "FaceDetectionShortRangeCpu",
deps = [
":face_detection_front_common",
":face_detection_short_range_common",
"//mediapipe/calculators/tensor:image_to_tensor_calculator",
"//mediapipe/calculators/tensor:inference_calculator",
"//mediapipe/calculators/util:to_image_calculator",
@@ -58,11 +58,11 @@ mediapipe_simple_subgraph(
)
mediapipe_simple_subgraph(
name = "face_detection_front_gpu",
graph = "face_detection_front_gpu.pbtxt",
register_as = "FaceDetectionFrontGpu",
name = "face_detection_short_range_gpu",
graph = "face_detection_short_range_gpu.pbtxt",
register_as = "FaceDetectionShortRangeGpu",
deps = [
":face_detection_front_common",
":face_detection_short_range_common",
"//mediapipe/calculators/tensor:image_to_tensor_calculator",
"//mediapipe/calculators/tensor:inference_calculator",
"//mediapipe/calculators/util:to_image_calculator",
@@ -70,9 +70,45 @@ mediapipe_simple_subgraph(
)
mediapipe_simple_subgraph(
name = "face_detection_front_common",
graph = "face_detection_front_common.pbtxt",
register_as = "FaceDetectionFrontCommon",
name = "face_detection_short_range_common",
graph = "face_detection_short_range_common.pbtxt",
register_as = "FaceDetectionShortRangeCommon",
deps = [
"//mediapipe/calculators/tensor:tensors_to_detections_calculator",
"//mediapipe/calculators/tflite:ssd_anchors_calculator",
"//mediapipe/calculators/util:detection_projection_calculator",
"//mediapipe/calculators/util:non_max_suppression_calculator",
],
)
mediapipe_simple_subgraph(
name = "face_detection_full_range_cpu",
graph = "face_detection_full_range_cpu.pbtxt",
register_as = "FaceDetectionFullRangeCpu",
deps = [
":face_detection_full_range_common",
"//mediapipe/calculators/tensor:image_to_tensor_calculator",
"//mediapipe/calculators/tensor:inference_calculator",
"//mediapipe/calculators/util:to_image_calculator",
],
)
mediapipe_simple_subgraph(
name = "face_detection_full_range_gpu",
graph = "face_detection_full_range_gpu.pbtxt",
register_as = "FaceDetectionFullRangeGpu",
deps = [
":face_detection_full_range_common",
"//mediapipe/calculators/tensor:image_to_tensor_calculator",
"//mediapipe/calculators/tensor:inference_calculator",
"//mediapipe/calculators/util:to_image_calculator",
],
)
mediapipe_simple_subgraph(
name = "face_detection_full_range_common",
graph = "face_detection_full_range_common.pbtxt",
register_as = "FaceDetectionFullRangeCommon",
deps = [
"//mediapipe/calculators/tensor:tensors_to_detections_calculator",
"//mediapipe/calculators/tflite:ssd_anchors_calculator",
@@ -83,8 +119,8 @@ mediapipe_simple_subgraph(
exports_files(
srcs = [
"face_detection_back.tflite",
"face_detection_back_sparse.tflite",
"face_detection_front.tflite",
"face_detection_full_range.tflite",
"face_detection_full_range_sparse.tflite",
"face_detection_short_range.tflite",
],
)
mediapipe/modules/face_detection/README.md
+4 -3
View File
@@ -2,6 +2,7 @@
Subgraphs|Details
:--- | :---
[`FaceDetectionFrontCpu`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_detection/face_detection_front_cpu.pbtxt)| Detects faces. Works best for images from front-facing cameras (i.e. selfie images). (CPU input, and inference is executed on CPU.)
[`FaceDetectionFrontGpu`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_detection/face_detection_front_gpu.pbtxt)| Detects faces. Works best for images from front-facing cameras (i.e. selfie images). (GPU input, and inference is executed on GPU.)
[`FaceDetectionFullRangeCpu`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_detection/face_detection_full_range_cpu.pbtxt)| Detects faces. Works best for faces within 5 meters from the camera. (CPU input, and inference is executed on CPU.)
[`FaceDetectionFullRangeGpu`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_detection/face_detection_full_range_gpu.pbtxt)| Detects faces. Works best for faces within 5 meters from the camera. (GPU input, and inference is executed on GPU.)
[`FaceDetectionShortRangeCpu`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_detection/face_detection_short_range_cpu.pbtxt)| Detects faces. Works best for faces within 2 meters from the camera. (CPU input, and inference is executed on CPU.)
[`FaceDetectionShortRangeGpu`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_detection/face_detection_short_range_gpu.pbtxt)| Detects faces. Works best for faces within 2 meters from the camera. (GPU input, and inference is executed on GPU.)
mediapipe/modules/face_detection/face_detection_back.tflite → mediapipe/modules/face_detection/face_detection_full_range.tflite
View File
mediapipe/modules/face_detection/face_detection_full_range_common.pbtxt
+102
View File
@@ -0,0 +1,102 @@
# MediaPipe graph performing common processing to detect faces using
# face_detection_full_range_sparse.tflite model, currently consisting of tensor
# post processing.
#
# EXAMPLE:
# node {
# calculator: "FaceDetectionFullRangeCommon"
# input_stream: "TENSORS:detection_tensors"
# input_stream: "MATRIX:transform_matrix"
# output_stream: "DETECTIONS:detections"
# }
type: "FaceDetectionShortRangeCommon"
# Detection tensors. (std::vector<Tensor>)
input_stream: "TENSORS:detection_tensors"
# A 4x4 row-major-order matrix that maps a point represented in the detection
# tensors to a desired coordinate system, e.g., in the original input image
# before scaling/cropping. (std::array<float, 16>)
input_stream: "MATRIX:transform_matrix"
# Detected faces. (std::vector<Detection>)
# NOTE: there will not be an output packet in the DETECTIONS stream for this
# particular timestamp if none of faces detected. However, the MediaPipe
# framework will internally inform the downstream calculators of the absence of
# this packet so that they don't wait for it unnecessarily.
output_stream: "DETECTIONS:detections"
# Generates a single side packet containing a vector of SSD anchors based on
# the specification in the options.
node {
calculator: "SsdAnchorsCalculator"
output_side_packet: "anchors"
options: {
[mediapipe.SsdAnchorsCalculatorOptions.ext] {
num_layers: 1
min_scale: 0.1484375
max_scale: 0.75
input_size_height: 192
input_size_width: 192
anchor_offset_x: 0.5
anchor_offset_y: 0.5
strides: 4
aspect_ratios: 1.0
fixed_anchor_size: true
interpolated_scale_aspect_ratio: 0.0
}
}
}
# Decodes the detection tensors generated by the TensorFlow Lite model, based on
# the SSD anchors and the specification in the options, into a vector of
# detections. Each detection describes a detected object.
node {
calculator: "TensorsToDetectionsCalculator"
input_stream: "TENSORS:detection_tensors"
input_side_packet: "ANCHORS:anchors"
output_stream: "DETECTIONS:unfiltered_detections"
options: {
[mediapipe.TensorsToDetectionsCalculatorOptions.ext] {
num_classes: 1
num_boxes: 2304
num_coords: 16
box_coord_offset: 0
keypoint_coord_offset: 4
num_keypoints: 6
num_values_per_keypoint: 2
sigmoid_score: true
score_clipping_thresh: 100.0
reverse_output_order: true
x_scale: 192.0
y_scale: 192.0
h_scale: 192.0
w_scale: 192.0
min_score_thresh: 0.6
}
}
}
# Performs non-max suppression to remove excessive detections.
node {
calculator: "NonMaxSuppressionCalculator"
input_stream: "unfiltered_detections"
output_stream: "filtered_detections"
options: {
[mediapipe.NonMaxSuppressionCalculatorOptions.ext] {
min_suppression_threshold: 0.3
overlap_type: INTERSECTION_OVER_UNION
algorithm: WEIGHTED
}
}
}
# Projects the detections from input tensor to the corresponding locations on
# the original image (input to the graph).
node {
calculator: "DetectionProjectionCalculator"
input_stream: "DETECTIONS:filtered_detections"
input_stream: "PROJECTION_MATRIX:transform_matrix"
output_stream: "DETECTIONS:detections"
}
mediapipe/modules/face_detection/face_detection_full_range_cpu.pbtxt
+79
View File
@@ -0,0 +1,79 @@
# MediaPipe graph to detect faces. (CPU input, and inference is executed on
# CPU.)
#
# It is required that "face_detection_full_range_sparse.tflite" is available at
# "mediapipe/modules/face_detection/face_detection_full_range_sparse.tflite"
# path during execution.
#
# EXAMPLE:
# node {
# calculator: "FaceDetectionFullRangeCpu"
# input_stream: "IMAGE:image"
# output_stream: "DETECTIONS:face_detections"
# }
type: "FaceDetectionFullRangeCpu"
# CPU image. (ImageFrame)
input_stream: "IMAGE:image"
# Detected faces. (std::vector<Detection>)
# NOTE: there will not be an output packet in the DETECTIONS stream for this
# particular timestamp if none of faces detected. However, the MediaPipe
# framework will internally inform the downstream calculators of the absence of
# this packet so that they don't wait for it unnecessarily.
output_stream: "DETECTIONS:detections"
# Converts the input CPU image (ImageFrame) to the multi-backend image type
# (Image).
node: {
calculator: "ToImageCalculator"
input_stream: "IMAGE_CPU:image"
output_stream: "IMAGE:multi_backend_image"
}
# Transforms the input image into a 192x192 tensor while keeping the aspect
# ratio (what is expected by the corresponding face detection model), resulting
# in potential letterboxing in the transformed image.
node: {
calculator: "ImageToTensorCalculator"
input_stream: "IMAGE:multi_backend_image"
output_stream: "TENSORS:input_tensors"
output_stream: "MATRIX:transform_matrix"
options: {
[mediapipe.ImageToTensorCalculatorOptions.ext] {
output_tensor_width: 192
output_tensor_height: 192
keep_aspect_ratio: true
output_tensor_float_range {
min: -1.0
max: 1.0
}
border_mode: BORDER_ZERO
}
}
}
# Runs a TensorFlow Lite model on CPU that takes an image tensor and outputs a
# vector of tensors representing, for instance, detection boxes/keypoints and
# scores.
node {
calculator: "InferenceCalculator"
input_stream: "TENSORS:input_tensors"
output_stream: "TENSORS:detection_tensors"
options: {
[mediapipe.InferenceCalculatorOptions.ext] {
model_path: "mediapipe/modules/face_detection/face_detection_full_range_sparse.tflite"
delegate { xnnpack {} }
}
#
}
}
# Performs tensor post processing to generate face detections.
node {
calculator: "FaceDetectionFullRangeCommon"
input_stream: "TENSORS:detection_tensors"
input_stream: "MATRIX:transform_matrix"
output_stream: "DETECTIONS:detections"
}
mediapipe/modules/face_detection/face_detection_full_range_gpu.pbtxt
+80
View File
@@ -0,0 +1,80 @@
# MediaPipe graph to detect faces. (GPU input, and inference is executed on
# GPU.)
#
# It is required that "face_detection_full_range_sparse.tflite" is available at
# "mediapipe/modules/face_detection/face_detection_full_range_sparse.tflite"
# path during execution.
#
# EXAMPLE:
# node {
# calculator: "FaceDetectionFullRangeGpu"
# input_stream: "IMAGE:image"
# output_stream: "DETECTIONS:face_detections"
# }
type: "FaceDetectionFullRangeGpu"
# GPU image. (GpuBuffer)
input_stream: "IMAGE:image"
# Detected faces. (std::vector<Detection>)
# NOTE: there will not be an output packet in the DETECTIONS stream for this
# particular timestamp if none of faces detected. However, the MediaPipe
# framework will internally inform the downstream calculators of the absence of
# this packet so that they don't wait for it unnecessarily.
output_stream: "DETECTIONS:detections"
# Converts the input GPU image (GpuBuffer) to the multi-backend image type
# (Image).
node: {
calculator: "ToImageCalculator"
input_stream: "IMAGE_GPU:image"
output_stream: "IMAGE:multi_backend_image"
}
# Transforms the input image into a 128x128 tensor while keeping the aspect
# ratio (what is expected by the corresponding face detection model), resulting
# in potential letterboxing in the transformed image.
node: {
calculator: "ImageToTensorCalculator"
input_stream: "IMAGE:multi_backend_image"
output_stream: "TENSORS:input_tensors"
output_stream: "MATRIX:transform_matrix"
options: {
[mediapipe.ImageToTensorCalculatorOptions.ext] {
output_tensor_width: 192
output_tensor_height: 192
keep_aspect_ratio: true
output_tensor_float_range {
min: -1.0
max: 1.0
}
border_mode: BORDER_ZERO
gpu_origin: TOP_LEFT
}
}
}
# Runs a TensorFlow Lite model on GPU that takes an image tensor and outputs a
# vector of tensors representing, for instance, detection boxes/keypoints and
# scores.
node {
calculator: "InferenceCalculator"
input_stream: "TENSORS:input_tensors"
output_stream: "TENSORS:detection_tensors"
options: {
[mediapipe.InferenceCalculatorOptions.ext] {
model_path: "mediapipe/modules/face_detection/face_detection_full_range_sparse.tflite"
#
delegate: { gpu { use_advanced_gpu_api: true } }
}
}
}
# Performs tensor post processing to generate face detections.
node {
calculator: "FaceDetectionFullRangeCommon"
input_stream: "TENSORS:detection_tensors"
input_stream: "MATRIX:transform_matrix"
output_stream: "DETECTIONS:detections"
}
mediapipe/modules/face_detection/face_detection_back_sparse.tflite → mediapipe/modules/face_detection/face_detection_full_range_sparse.tflite
View File
mediapipe/modules/face_detection/face_detection_front.tflite → mediapipe/modules/face_detection/face_detection_short_range.tflite
View File
mediapipe/modules/face_detection/face_detection_front_by_roi_cpu.pbtxt → mediapipe/modules/face_detection/face_detection_short_range_by_roi_cpu.pbtxt
+6 -6
View File
@@ -1,19 +1,19 @@
# MediaPipe graph to detect faces. (CPU input, and inference is executed on
# CPU.)
#
# It is required that "face_detection_front.tflite" is available at
# "mediapipe/modules/face_detection/face_detection_front.tflite"
# It is required that "face_detection_short_range.tflite" is available at
# "mediapipe/modules/face_detection/face_detection_short_range.tflite"
# path during execution.
#
# EXAMPLE:
# node {
# calculator: "FaceDetectionFrontByRoiCpu"
# calculator: "FaceDetectionShortRangeByRoiCpu"
# input_stream: "IMAGE:image"
# input_stream: "ROI:roi"
# output_stream: "DETECTIONS:face_detections"
# }
type: "FaceDetectionFrontByRoiCpu"
type: "FaceDetectionShortRangeByRoiCpu"
# CPU image. (ImageFrame)
input_stream: "IMAGE:image"
@@ -68,7 +68,7 @@ node {
output_stream: "TENSORS:detection_tensors"
options: {
[mediapipe.InferenceCalculatorOptions.ext] {
model_path: "mediapipe/modules/face_detection/face_detection_front.tflite"
model_path: "mediapipe/modules/face_detection/face_detection_short_range.tflite"
delegate { xnnpack {} }
}
}
@@ -76,7 +76,7 @@ node {
# Performs tensor post processing to generate face detections.
node {
calculator: "FaceDetectionFrontCommon"
calculator: "FaceDetectionShortRangeCommon"
input_stream: "TENSORS:detection_tensors"
input_stream: "MATRIX:transform_matrix"
output_stream: "DETECTIONS:detections"
mediapipe/modules/face_detection/face_detection_front_by_roi_gpu.pbtxt → mediapipe/modules/face_detection/face_detection_short_range_by_roi_gpu.pbtxt
+6 -6
View File
@@ -1,19 +1,19 @@
# MediaPipe graph to detect faces. (CPU input, and inference is executed on
# CPU.)
#
# It is required that "face_detection_front.tflite" is available at
# "mediapipe/modules/face_detection/face_detection_front.tflite"
# It is required that "face_detection_short_range.tflite" is available at
# "mediapipe/modules/face_detection/face_detection_short_range.tflite"
# path during execution.
#
# EXAMPLE:
# node {
# calculator: "FaceDetectionFrontByRoiGpu"
# calculator: "FaceDetectionShortRangeByRoiGpu"
# input_stream: "IMAGE:image"
# input_stream: "ROI:roi"
# output_stream: "DETECTIONS:face_detections"
# }
type: "FaceDetectionFrontByRoiGpu"
type: "FaceDetectionShortRangeByRoiGpu"
# GPU image. (GpuBuffer)
input_stream: "IMAGE:image"
@@ -69,14 +69,14 @@ node {
output_stream: "TENSORS:detection_tensors"
options: {
[mediapipe.InferenceCalculatorOptions.ext] {
model_path: "mediapipe/modules/face_detection/face_detection_front.tflite"
model_path: "mediapipe/modules/face_detection/face_detection_short_range.tflite"
}
}
}
# Performs tensor post processing to generate face detections.
node {
calculator: "FaceDetectionFrontCommon"
calculator: "FaceDetectionShortRangeCommon"
input_stream: "TENSORS:detection_tensors"
input_stream: "MATRIX:transform_matrix"
output_stream: "DETECTIONS:detections"
mediapipe/modules/face_detection/face_detection_front_common.pbtxt → mediapipe/modules/face_detection/face_detection_short_range_common.pbtxt
+2 -2
View File
@@ -3,13 +3,13 @@
#
# EXAMPLE:
# node {
# calculator: "FaceDetectionFrontCommon"
# calculator: "FaceDetectionShortRangeCommon"
# input_stream: "TENSORS:detection_tensors"
# input_stream: "MATRIX:transform_matrix"
# output_stream: "DETECTIONS:detections"
# }
type: "FaceDetectionFrontCommon"
type: "FaceDetectionShortRangeCommon"
# Detection tensors. (std::vector<Tensor>)
input_stream: "TENSORS:detection_tensors"
mediapipe/modules/face_detection/face_detection_front_cpu.pbtxt → mediapipe/modules/face_detection/face_detection_short_range_cpu.pbtxt
+6 -6
View File
@@ -1,18 +1,18 @@
# MediaPipe graph to detect faces. (CPU input, and inference is executed on
# CPU.)
#
# It is required that "face_detection_front.tflite" is available at
# "mediapipe/modules/face_detection/face_detection_front.tflite"
# It is required that "face_detection_short_range.tflite" is available at
# "mediapipe/modules/face_detection/face_detection_short_range.tflite"
# path during execution.
#
# EXAMPLE:
# node {
# calculator: "FaceDetectionFrontCpu"
# calculator: "FaceDetectionShortRangeCpu"
# input_stream: "IMAGE:image"
# output_stream: "DETECTIONS:face_detections"
# }
type: "FaceDetectionFrontCpu"
type: "FaceDetectionShortRangeCpu"
# CPU image. (ImageFrame)
input_stream: "IMAGE:image"
@@ -63,7 +63,7 @@ node {
output_stream: "TENSORS:detection_tensors"
options: {
[mediapipe.InferenceCalculatorOptions.ext] {
model_path: "mediapipe/modules/face_detection/face_detection_front.tflite"
model_path: "mediapipe/modules/face_detection/face_detection_short_range.tflite"
delegate { xnnpack {} }
}
}
@@ -71,7 +71,7 @@ node {
# Performs tensor post processing to generate face detections.
node {
calculator: "FaceDetectionFrontCommon"
calculator: "FaceDetectionShortRangeCommon"
input_stream: "TENSORS:detection_tensors"
input_stream: "MATRIX:transform_matrix"
output_stream: "DETECTIONS:detections"
mediapipe/modules/face_detection/face_detection_front_gpu.pbtxt → mediapipe/modules/face_detection/face_detection_short_range_gpu.pbtxt
+6 -6
View File
@@ -1,18 +1,18 @@
# MediaPipe graph to detect faces. (CPU input, and inference is executed on
# CPU.)
#
# It is required that "face_detection_front.tflite" is available at
# "mediapipe/modules/face_detection/face_detection_front.tflite"
# It is required that "face_detection_short_range.tflite" is available at
# "mediapipe/modules/face_detection/face_detection_short_range.tflite"
# path during execution.
#
# EXAMPLE:
# node {
# calculator: "FaceDetectionFrontGpu"
# calculator: "FaceDetectionShortRangeGpu"
# input_stream: "IMAGE:image"
# output_stream: "DETECTIONS:face_detections"
# }
type: "FaceDetectionFrontGpu"
type: "FaceDetectionShortRangeGpu"
# GPU image. (GpuBuffer)
input_stream: "IMAGE:image"
@@ -64,14 +64,14 @@ node {
output_stream: "TENSORS:detection_tensors"
options: {
[mediapipe.InferenceCalculatorOptions.ext] {
model_path: "mediapipe/modules/face_detection/face_detection_front.tflite"
model_path: "mediapipe/modules/face_detection/face_detection_short_range.tflite"
}
}
}
# Performs tensor post processing to generate face detections.
node {
calculator: "FaceDetectionFrontCommon"
calculator: "FaceDetectionShortRangeCommon"
input_stream: "TENSORS:detection_tensors"
input_stream: "MATRIX:transform_matrix"
output_stream: "DETECTIONS:detections"
mediapipe/modules/face_landmark/BUILD
+2 -2
View File
@@ -70,7 +70,7 @@ mediapipe_simple_subgraph(
"//mediapipe/calculators/image:image_properties_calculator",
"//mediapipe/calculators/util:association_norm_rect_calculator",
"//mediapipe/calculators/util:collection_has_min_size_calculator",
"//mediapipe/modules/face_detection:face_detection_front_cpu",
"//mediapipe/modules/face_detection:face_detection_short_range_cpu",
],
)
@@ -91,7 +91,7 @@ mediapipe_simple_subgraph(
"//mediapipe/calculators/image:image_properties_calculator",
"//mediapipe/calculators/util:association_norm_rect_calculator",
"//mediapipe/calculators/util:collection_has_min_size_calculator",
"//mediapipe/modules/face_detection:face_detection_front_gpu",
"//mediapipe/modules/face_detection:face_detection_short_range_gpu",
],
)
mediapipe/modules/face_landmark/face_detection_front_detection_to_roi.pbtxt
+1 -1
View File
@@ -1,6 +1,6 @@
# MediaPipe graph to calculate face region of interest (ROI) from the very
# first face detection in the vector of detections provided by
# "FaceDetectionFrontCpu" or "FaceDetectionFrontGpu"
# "FaceDetectionShortRangeCpu" or "FaceDetectionShortRangeGpu"
#
# NOTE: this graph is subject to change and should not be used directly.
mediapipe/modules/face_landmark/face_landmark_front_cpu.pbtxt
+3 -3
View File
@@ -2,8 +2,8 @@
# executed on CPU.) This graph tries to skip face detection as much as possible
# by using previously detected/predicted landmarks for new images.
#
# It is required that "face_detection_front.tflite" is available at
# "mediapipe/modules/face_detection/face_detection_front.tflite"
# It is required that "face_detection_short_range.tflite" is available at
# "mediapipe/modules/face_detection/face_detection_short_range.tflite"
# path during execution.
#
# It is required that "face_landmark.tflite" is available at
@@ -89,7 +89,7 @@ node {
# Detects faces.
node {
calculator: "FaceDetectionFrontCpu"
calculator: "FaceDetectionShortRangeCpu"
input_stream: "IMAGE:gated_image"
output_stream: "DETECTIONS:all_face_detections"
}
mediapipe/modules/face_landmark/face_landmark_front_gpu.pbtxt
+3 -3
View File
@@ -2,8 +2,8 @@
# executed on GPU.) This graph tries to skip face detection as much as possible
# by using previously detected/predicted landmarks for new images.
#
# It is required that "face_detection_front.tflite" is available at
# "mediapipe/modules/face_detection/face_detection_front.tflite"
# It is required that "face_detection_short_range.tflite" is available at
# "mediapipe/modules/face_detection/face_detection_short_range.tflite"
# path during execution.
#
# It is required that "face_landmark.tflite" is available at
@@ -89,7 +89,7 @@ node {
# Detects faces.
node {
calculator: "FaceDetectionFrontGpu"
calculator: "FaceDetectionShortRangeGpu"
input_stream: "IMAGE:gated_image"
output_stream: "DETECTIONS:all_face_detections"
}
mediapipe/modules/face_landmark/face_landmark_front_side_model_cpu.pbtxt
+224
View File
@@ -0,0 +1,224 @@
# MediaPipe graph to detect/predict face landmarks. (CPU input, and inference is
# executed on CPU.) This graph tries to skip face detection as much as possible
# by using previously detected/predicted landmarks for new images.
#
# EXAMPLE:
# node {
# calculator: "FaceLandmarkFrontSideModelCpu"
# input_stream: "IMAGE:image"
# input_side_packet: "NUM_FACES:num_faces"
# input_side_packet: "MODEL:0:face_detection_model"
# input_side_packet: "MODEL:1:face_landmark_model"
# output_stream: "LANDMARKS:multi_face_landmarks"
# }
type: "FaceLandmarkFrontSideModelCpu"
# CPU image. (ImageFrame)
input_stream: "IMAGE:image"
# Max number of faces to detect/track. (int)
input_side_packet: "NUM_FACES:num_faces"
# TfLite model to detect faces.
# (std::unique_ptr<tflite::FlatBufferModel,
# std::function<void(tflite::FlatBufferModel*)>>)
# NOTE: mediapipe/modules/face_detection/face_detection_short_range.tflite
# model only, can be passed here, otherwise - results are undefined.
input_side_packet: "MODEL:0:face_detection_model"
# TfLite model to detect face landmarks.
# (std::unique_ptr<tflite::FlatBufferModel,
# std::function<void(tflite::FlatBufferModel*)>>)
# NOTE: mediapipe/modules/face_landmark/face_landmark.tflite model
# only, can be passed here, otherwise - results are undefined.
input_side_packet: "MODEL:1:face_landmark_model"
# Collection of detected/predicted faces, each represented as a list of 468 face
# landmarks. (std::vector<NormalizedLandmarkList>)
# NOTE: there will not be an output packet in the LANDMARKS stream for this
# particular timestamp if none of faces detected. However, the MediaPipe
# framework will internally inform the downstream calculators of the absence of
# this packet so that they don't wait for it unnecessarily.
output_stream: "LANDMARKS:multi_face_landmarks"
# Extra outputs (for debugging, for instance).
# Detected faces. (std::vector<Detection>)
output_stream: "DETECTIONS:face_detections"
# Regions of interest calculated based on landmarks.
# (std::vector<NormalizedRect>)
output_stream: "ROIS_FROM_LANDMARKS:face_rects_from_landmarks"
# Regions of interest calculated based on face detections.
# (std::vector<NormalizedRect>)
output_stream: "ROIS_FROM_DETECTIONS:face_rects_from_detections"
# Determines if an input vector of NormalizedRect has a size greater than or
# equal to the provided num_faces.
node {
calculator: "NormalizedRectVectorHasMinSizeCalculator"
input_stream: "ITERABLE:prev_face_rects_from_landmarks"
input_side_packet: "num_faces"
output_stream: "prev_has_enough_faces"
}
# Drops the incoming image if FaceLandmarkCpu was able to identify face presence
# in the previous image. Otherwise, passes the incoming image through to trigger
# a new round of face detection in FaceDetectionShortRangeCpu.
node {
calculator: "GateCalculator"
input_stream: "image"
input_stream: "DISALLOW:prev_has_enough_faces"
output_stream: "gated_image"
options: {
[mediapipe.GateCalculatorOptions.ext] {
empty_packets_as_allow: true
}
}
}
# Detects faces.
node {
calculator: "FaceDetectionShortRangeSideModelCpu"
input_stream: "IMAGE:gated_image"
input_side_packet: "MODEL:face_detection_model"
output_stream: "DETECTIONS:all_face_detections"
}
# Makes sure there are no more detections than the provided num_faces.
node {
calculator: "ClipDetectionVectorSizeCalculator"
input_stream: "all_face_detections"
output_stream: "face_detections"
input_side_packet: "num_faces"
}
# Calculate size of the image.
node {
calculator: "ImagePropertiesCalculator"
input_stream: "IMAGE:gated_image"
output_stream: "SIZE:gated_image_size"
}
# Outputs each element of face_detections at a fake timestamp for the rest of
# the graph to process. Clones the image size packet for each face_detection at
# the fake timestamp. At the end of the loop, outputs the BATCH_END timestamp
# for downstream calculators to inform them that all elements in the vector have
# been processed.
node {
calculator: "BeginLoopDetectionCalculator"
input_stream: "ITERABLE:face_detections"
input_stream: "CLONE:gated_image_size"
output_stream: "ITEM:face_detection"
output_stream: "CLONE:detections_loop_image_size"
output_stream: "BATCH_END:detections_loop_end_timestamp"
}
# Calculates region of interest based on face detections, so that can be used
# to detect landmarks.
node {
calculator: "FaceDetectionFrontDetectionToRoi"
input_stream: "DETECTION:face_detection"
input_stream: "IMAGE_SIZE:detections_loop_image_size"
output_stream: "ROI:face_rect_from_detection"
}
# Collects a NormalizedRect for each face into a vector. Upon receiving the
# BATCH_END timestamp, outputs the vector of NormalizedRect at the BATCH_END
# timestamp.
node {
calculator: "EndLoopNormalizedRectCalculator"
input_stream: "ITEM:face_rect_from_detection"
input_stream: "BATCH_END:detections_loop_end_timestamp"
output_stream: "ITERABLE:face_rects_from_detections"
}
# Performs association between NormalizedRect vector elements from previous
# image and rects based on face detections from the current image. This
# calculator ensures that the output face_rects vector doesn't contain
# overlapping regions based on the specified min_similarity_threshold.
node {
calculator: "AssociationNormRectCalculator"
input_stream: "face_rects_from_detections"
input_stream: "prev_face_rects_from_landmarks"
output_stream: "face_rects"
options: {
[mediapipe.AssociationCalculatorOptions.ext] {
min_similarity_threshold: 0.5
}
}
}
# Calculate size of the image.
node {
calculator: "ImagePropertiesCalculator"
input_stream: "IMAGE:image"
output_stream: "SIZE:image_size"
}
# Outputs each element of face_rects at a fake timestamp for the rest of the
# graph to process. Clones image and image size packets for each
# single_face_rect at the fake timestamp. At the end of the loop, outputs the
# BATCH_END timestamp for downstream calculators to inform them that all
# elements in the vector have been processed.
node {
calculator: "BeginLoopNormalizedRectCalculator"
input_stream: "ITERABLE:face_rects"
input_stream: "CLONE:0:image"
input_stream: "CLONE:1:image_size"
output_stream: "ITEM:face_rect"
output_stream: "CLONE:0:landmarks_loop_image"
output_stream: "CLONE:1:landmarks_loop_image_size"
output_stream: "BATCH_END:landmarks_loop_end_timestamp"
}
# Detects face landmarks within specified region of interest of the image.
node {
calculator: "FaceLandmarkSideModelCpu"
input_stream: "IMAGE:landmarks_loop_image"
input_stream: "ROI:face_rect"
input_side_packet: "MODEL:face_landmark_model"
output_stream: "LANDMARKS:face_landmarks"
}
# Calculates region of interest based on face landmarks, so that can be reused
# for subsequent image.
node {
calculator: "FaceLandmarkLandmarksToRoi"
input_stream: "LANDMARKS:face_landmarks"
input_stream: "IMAGE_SIZE:landmarks_loop_image_size"
output_stream: "ROI:face_rect_from_landmarks"
}
# Collects a set of landmarks for each face into a vector. Upon receiving the
# BATCH_END timestamp, outputs the vector of landmarks at the BATCH_END
# timestamp.
node {
calculator: "EndLoopNormalizedLandmarkListVectorCalculator"
input_stream: "ITEM:face_landmarks"
input_stream: "BATCH_END:landmarks_loop_end_timestamp"
output_stream: "ITERABLE:multi_face_landmarks"
}
# Collects a NormalizedRect for each face into a vector. Upon receiving the
# BATCH_END timestamp, outputs the vector of NormalizedRect at the BATCH_END
# timestamp.
node {
calculator: "EndLoopNormalizedRectCalculator"
input_stream: "ITEM:face_rect_from_landmarks"
input_stream: "BATCH_END:landmarks_loop_end_timestamp"
output_stream: "ITERABLE:face_rects_from_landmarks"
}
# Caches face rects calculated from landmarks, and upon the arrival of the next
# input image, sends out the cached rects with timestamps replaced by that of
# the input image, essentially generating a packet that carries the previous
# face rects. Note that upon the arrival of the very first input image, a
# timestamp bound update occurs to jump start the feedback loop.
node {
calculator: "PreviousLoopbackCalculator"
input_stream: "MAIN:image"
input_stream: "LOOP:face_rects_from_landmarks"
input_stream_info: {
tag_index: "LOOP"
back_edge: true
}
output_stream: "PREV_LOOP:prev_face_rects_from_landmarks"
}
mediapipe/modules/face_landmark/face_landmark_front_side_model_gpu.pbtxt
+224
View File
@@ -0,0 +1,224 @@
# MediaPipe graph to detect/predict face landmarks. (GPU input, and inference is
# executed on GPU.) This graph tries to skip face detection as much as possible
# by using previously detected/predicted landmarks for new images.
#
# EXAMPLE:
# node {
# calculator: "FaceLandmarkFrontSideModelGpu"
# input_stream: "IMAGE:image"
# input_side_packet: "NUM_FACES:num_faces"
# input_side_packet: "MODEL:0:face_detection_model"
# input_side_packet: "MODEL:1:face_landmark_model"
# output_stream: "LANDMARKS:multi_face_landmarks"
# }
type: "FaceLandmarkFrontSideModelGpu"
# GPU image. (GpuBuffer)
input_stream: "IMAGE:image"
# Max number of faces to detect/track. (int)
input_side_packet: "NUM_FACES:num_faces"
# TfLite model to detect faces.
# (std::unique_ptr<tflite::FlatBufferModel,
# std::function<void(tflite::FlatBufferModel*)>>)
# NOTE: mediapipe/modules/face_detection/face_detection_short_range.tflite
# model only, can be passed here, otherwise - results are undefined.
input_side_packet: "MODEL:0:face_detection_model"
# TfLite model to detect face landmarks.
# (std::unique_ptr<tflite::FlatBufferModel,
# std::function<void(tflite::FlatBufferModel*)>>)
# NOTE: mediapipe/modules/face_landmark/face_landmark.tflite model
# only, can be passed here, otherwise - results are undefined.
input_side_packet: "MODEL:1:face_landmark_model"
# Collection of detected/predicted faces, each represented as a list of 468 face
# landmarks. (std::vector<NormalizedLandmarkList>)
# NOTE: there will not be an output packet in the LANDMARKS stream for this
# particular timestamp if none of faces detected. However, the MediaPipe
# framework will internally inform the downstream calculators of the absence of
# this packet so that they don't wait for it unnecessarily.
output_stream: "LANDMARKS:multi_face_landmarks"
# Extra outputs (for debugging, for instance).
# Detected faces. (std::vector<Detection>)
output_stream: "DETECTIONS:face_detections"
# Regions of interest calculated based on landmarks.
# (std::vector<NormalizedRect>)
output_stream: "ROIS_FROM_LANDMARKS:face_rects_from_landmarks"
# Regions of interest calculated based on face detections.
# (std::vector<NormalizedRect>)
output_stream: "ROIS_FROM_DETECTIONS:face_rects_from_detections"
# Determines if an input vector of NormalizedRect has a size greater than or
# equal to the provided num_faces.
node {
calculator: "NormalizedRectVectorHasMinSizeCalculator"
input_stream: "ITERABLE:prev_face_rects_from_landmarks"
input_side_packet: "num_faces"
output_stream: "prev_has_enough_faces"
}
# Drops the incoming image if FaceLandmarkGpu was able to identify face presence
# in the previous image. Otherwise, passes the incoming image through to trigger
# a new round of face detection in FaceDetectionShortRangeGpu.
node {
calculator: "GateCalculator"
input_stream: "image"
input_stream: "DISALLOW:prev_has_enough_faces"
output_stream: "gated_image"
options: {
[mediapipe.GateCalculatorOptions.ext] {
empty_packets_as_allow: true
}
}
}
# Detects faces.
node {
calculator: "FaceDetectionShortRangeSideModelGpu"
input_stream: "IMAGE:gated_image"
input_side_packet: "MODEL:face_detection_model"
output_stream: "DETECTIONS:all_face_detections"
}
# Makes sure there are no more detections than the provided num_faces.
node {
calculator: "ClipDetectionVectorSizeCalculator"
input_stream: "all_face_detections"
output_stream: "face_detections"
input_side_packet: "num_faces"
}
# Calculate size of the image.
node {
calculator: "ImagePropertiesCalculator"
input_stream: "IMAGE_GPU:gated_image"
output_stream: "SIZE:gated_image_size"
}
# Outputs each element of face_detections at a fake timestamp for the rest of
# the graph to process. Clones the image size packet for each face_detection at
# the fake timestamp. At the end of the loop, outputs the BATCH_END timestamp
# for downstream calculators to inform them that all elements in the vector have
# been processed.
node {
calculator: "BeginLoopDetectionCalculator"
input_stream: "ITERABLE:face_detections"
input_stream: "CLONE:gated_image_size"
output_stream: "ITEM:face_detection"
output_stream: "CLONE:detections_loop_image_size"
output_stream: "BATCH_END:detections_loop_end_timestamp"
}
# Calculates region of interest based on face detections, so that can be used
# to detect landmarks.
node {
calculator: "FaceDetectionFrontDetectionToRoi"
input_stream: "DETECTION:face_detection"
input_stream: "IMAGE_SIZE:detections_loop_image_size"
output_stream: "ROI:face_rect_from_detection"
}
# Collects a NormalizedRect for each face into a vector. Upon receiving the
# BATCH_END timestamp, outputs the vector of NormalizedRect at the BATCH_END
# timestamp.
node {
calculator: "EndLoopNormalizedRectCalculator"
input_stream: "ITEM:face_rect_from_detection"
input_stream: "BATCH_END:detections_loop_end_timestamp"
output_stream: "ITERABLE:face_rects_from_detections"
}
# Performs association between NormalizedRect vector elements from previous
# image and rects based on face detections from the current image. This
# calculator ensures that the output face_rects vector doesn't contain
# overlapping regions based on the specified min_similarity_threshold.
node {
calculator: "AssociationNormRectCalculator"
input_stream: "face_rects_from_detections"
input_stream: "prev_face_rects_from_landmarks"
output_stream: "face_rects"
options: {
[mediapipe.AssociationCalculatorOptions.ext] {
min_similarity_threshold: 0.5
}
}
}
# Calculate size of the image.
node {
calculator: "ImagePropertiesCalculator"
input_stream: "IMAGE_GPU:image"
output_stream: "SIZE:image_size"
}
# Outputs each element of face_rects at a fake timestamp for the rest of the
# graph to process. Clones image and image size packets for each
# single_face_rect at the fake timestamp. At the end of the loop, outputs the
# BATCH_END timestamp for downstream calculators to inform them that all
# elements in the vector have been processed.
node {
calculator: "BeginLoopNormalizedRectCalculator"
input_stream: "ITERABLE:face_rects"
input_stream: "CLONE:0:image"
input_stream: "CLONE:1:image_size"
output_stream: "ITEM:face_rect"
output_stream: "CLONE:0:landmarks_loop_image"
output_stream: "CLONE:1:landmarks_loop_image_size"
output_stream: "BATCH_END:landmarks_loop_end_timestamp"
}
# Detects face landmarks within specified region of interest of the image.
node {
calculator: "FaceLandmarkSideModelGpu"
input_stream: "IMAGE:landmarks_loop_image"
input_stream: "ROI:face_rect"
input_side_packet: "MODEL:face_landmark_model"
output_stream: "LANDMARKS:face_landmarks"
}
# Calculates region of interest based on face landmarks, so that can be reused
# for subsequent image.
node {
calculator: "FaceLandmarkLandmarksToRoi"
input_stream: "LANDMARKS:face_landmarks"
input_stream: "IMAGE_SIZE:landmarks_loop_image_size"
output_stream: "ROI:face_rect_from_landmarks"
}
# Collects a set of landmarks for each face into a vector. Upon receiving the
# BATCH_END timestamp, outputs the vector of landmarks at the BATCH_END
# timestamp.
node {
calculator: "EndLoopNormalizedLandmarkListVectorCalculator"
input_stream: "ITEM:face_landmarks"
input_stream: "BATCH_END:landmarks_loop_end_timestamp"
output_stream: "ITERABLE:multi_face_landmarks"
}
# Collects a NormalizedRect for each face into a vector. Upon receiving the
# BATCH_END timestamp, outputs the vector of NormalizedRect at the BATCH_END
# timestamp.
node {
calculator: "EndLoopNormalizedRectCalculator"
input_stream: "ITEM:face_rect_from_landmarks"
input_stream: "BATCH_END:landmarks_loop_end_timestamp"
output_stream: "ITERABLE:face_rects_from_landmarks"
}
# Caches face rects calculated from landmarks, and upon the arrival of the next
# input image, sends out the cached rects with timestamps replaced by that of
# the input image, essentially generating a packet that carries the previous
# face rects. Note that upon the arrival of the very first input image, a
# timestamp bound update occurs to jump start the feedback loop.
node {
calculator: "PreviousLoopbackCalculator"
input_stream: "MAIN:image"
input_stream: "LOOP:face_rects_from_landmarks"
input_stream_info: {
tag_index: "LOOP"
back_edge: true
}
output_stream: "PREV_LOOP:prev_face_rects_from_landmarks"
}
mediapipe/modules/hand_landmark/BUILD
+21 -1
View File
@@ -86,6 +86,26 @@ mediapipe_simple_subgraph(
],
)
mediapipe_simple_subgraph(
name = "hand_landmark_tracking_cpu_image",
graph = "hand_landmark_tracking_cpu_image.pbtxt",
register_as = "HandLandmarkTrackingCpuImage",
deps = [
":hand_landmark_tracking_cpu",
"//mediapipe/calculators/util:from_image_calculator",
],
)
mediapipe_simple_subgraph(
name = "hand_landmark_tracking_gpu_image",
graph = "hand_landmark_tracking_gpu_image.pbtxt",
register_as = "HandLandmarkTrackingGpuImage",
deps = [
":hand_landmark_tracking_gpu",
"//mediapipe/calculators/util:from_image_calculator",
],
)
mediapipe_simple_subgraph(
name = "hand_landmark_tracking_cpu",
graph = "hand_landmark_tracking_cpu.pbtxt",
@@ -124,7 +144,7 @@ mediapipe_simple_subgraph(
graph = "hand_landmark_landmarks_to_roi.pbtxt",
register_as = "HandLandmarkLandmarksToRoi",
deps = [
"//mediapipe/calculators/core:split_normalized_landmark_list_calculator",
"//mediapipe/calculators/core:split_landmarks_calculator",
"//mediapipe/calculators/util:rect_transformation_calculator",
"//mediapipe/modules/hand_landmark/calculators:hand_landmarks_to_rect_calculator",
],
mediapipe/modules/hand_landmark/hand_landmark_tracking_cpu_image.pbtxt
+57
View File
@@ -0,0 +1,57 @@
# MediaPipe graph to detect/predict hand landmarks on CPU.
#
# The procedure is done in two steps:
# - locate palms/hands
# - detect landmarks for each palm/hand.
# This graph tries to skip palm detection as much as possible by reusing
# previously detected/predicted landmarks for new images.
type: "HandLandmarkTrackingCpuImage"
# Input image. (Image)
input_stream: "IMAGE:image"
# Max number of hands to detect/track. (int)
input_side_packet: "NUM_HANDS:num_hands"
# Collection of detected/predicted hands, each represented as a list of
# landmarks. (std::vector<NormalizedLandmarkList>)
# NOTE: there will not be an output packet in the LANDMARKS stream for this
# particular timestamp if none of hands detected. However, the MediaPipe
# framework will internally inform the downstream calculators of the absence of
# this packet so that they don't wait for it unnecessarily.
output_stream: "LANDMARKS:multi_hand_landmarks"
# Collection of handedness of the detected hands (i.e. is hand left or right),
# each represented as a Classification proto.
# Note that handedness is determined assuming the input image is mirrored,
# i.e., taken with a front-facing/selfie camera with images flipped
# horizontally.
output_stream: "HANDEDNESS:multi_handedness"
# Extra outputs (for debugging, for instance).
# Detected palms. (std::vector<Detection>)
output_stream: "PALM_DETECTIONS:palm_detections"
# Regions of interest calculated based on landmarks.
# (std::vector<NormalizedRect>)
output_stream: "HAND_ROIS_FROM_LANDMARKS:hand_rects"
# Regions of interest calculated based on palm detections.
# (std::vector<NormalizedRect>)
output_stream: "HAND_ROIS_FROM_PALM_DETECTIONS:hand_rects_from_palm_detections"
# Converts Image to GpuBuffer for HandLandmarkTrackingGpu to consume.
node {
calculator: "FromImageCalculator"
input_stream: "IMAGE:image"
output_stream: "IMAGE_CPU:image_frame"
}
node {
calculator: "HandLandmarkTrackingCpu"
input_stream: "IMAGE:image_frame"
input_side_packet: "NUM_HANDS:num_hands"
output_stream: "LANDMARKS:multi_hand_landmarks"
output_stream: "HANDEDNESS:multi_handedness"
output_stream: "PALM_DETECTIONS:palm_detections"
output_stream: "HAND_ROIS_FROM_LANDMARKS:hand_rects"
output_stream: "HAND_ROIS_FROM_PALM_DETECTIONS:hand_rects_from_palm_detections"
}
mediapipe/modules/hand_landmark/hand_landmark_tracking_gpu_image.pbtxt
+59
View File
@@ -0,0 +1,59 @@
# MediaPipe graph to detect/predict hand landmarks on GPU.
#
# The procedure is done in two steps:
# - locate palms/hands
# - detect landmarks for each palm/hand.
# This graph tries to skip palm detection as much as possible by reusing
# previously detected/predicted landmarks for new images.
type: "HandLandmarkTrackingGpuImage"
# Input image. (Image)
input_stream: "IMAGE:image"
# Max number of hands to detect/track. (int)
input_side_packet: "NUM_HANDS:num_hands"
# Collection of detected/predicted hands, each represented as a list of
# landmarks. (std::vector<NormalizedLandmarkList>)
# NOTE: there will not be an output packet in the LANDMARKS stream for this
# particular timestamp if none of hands detected. However, the MediaPipe
# framework will internally inform the downstream calculators of the absence of
# this packet so that they don't wait for it unnecessarily.
output_stream: "LANDMARKS:multi_hand_landmarks"
# Collection of handedness of the detected hands (i.e. is hand left or right),
# each represented as a Classification proto.
# Note that handedness is determined assuming the input image is mirrored,
# i.e., taken with a front-facing/selfie camera with images flipped
# horizontally.
output_stream: "HANDEDNESS:multi_handedness"
# The original input image. (Image)
output_stream: "IMAGE:image"
# Extra outputs (for debugging, for instance).
# Detected palms. (std::vector<Detection>)
output_stream: "PALM_DETECTIONS:palm_detections"
# Regions of interest calculated based on landmarks.
# (std::vector<NormalizedRect>)
output_stream: "HAND_ROIS_FROM_LANDMARKS:hand_rects"
# Regions of interest calculated based on palm detections.
# (std::vector<NormalizedRect>)
output_stream: "HAND_ROIS_FROM_PALM_DETECTIONS:hand_rects_from_palm_detections"
# Converts Image to GpuBuffer for HandLandmarkTrackingGpu to consume.
node {
calculator: "FromImageCalculator"
input_stream: "IMAGE:image"
output_stream: "IMAGE_GPU:gpu_buffer"
}
node {
calculator: "HandLandmarkTrackingGpu"
input_stream: "IMAGE:gpu_buffer"
input_side_packet: "NUM_HANDS:num_hands"
output_stream: "LANDMARKS:multi_hand_landmarks"
output_stream: "HANDEDNESS:multi_handedness"
output_stream: "PALM_DETECTIONS:palm_detections"
output_stream: "HAND_ROIS_FROM_LANDMARKS:hand_rects"
output_stream: "HAND_ROIS_FROM_PALM_DETECTIONS:hand_rects_from_palm_detections"
}
mediapipe/modules/holistic_landmark/BUILD
+9 -9
View File
@@ -31,9 +31,9 @@ mediapipe_simple_subgraph(
":face_detection_front_detections_to_roi",
":face_landmarks_from_pose_to_recrop_roi",
":face_tracking",
"//mediapipe/calculators/core:split_normalized_landmark_list_calculator",
"//mediapipe/calculators/core:split_landmarks_calculator",
"//mediapipe/calculators/image:image_properties_calculator",
"//mediapipe/modules/face_detection:face_detection_front_by_roi_gpu",
"//mediapipe/modules/face_detection:face_detection_short_range_by_roi_gpu",
"//mediapipe/modules/face_landmark:face_landmark_gpu",
],
)
@@ -46,9 +46,9 @@ mediapipe_simple_subgraph(
":face_detection_front_detections_to_roi",
":face_landmarks_from_pose_to_recrop_roi",
":face_tracking",
"//mediapipe/calculators/core:split_normalized_landmark_list_calculator",
"//mediapipe/calculators/core:split_landmarks_calculator",
"//mediapipe/calculators/image:image_properties_calculator",
"//mediapipe/modules/face_detection:face_detection_front_by_roi_cpu",
"//mediapipe/modules/face_detection:face_detection_short_range_by_roi_cpu",
"//mediapipe/modules/face_landmark:face_landmark_cpu",
],
)
@@ -131,7 +131,7 @@ mediapipe_simple_subgraph(
graph = "hand_landmarks_to_roi.pbtxt",
register_as = "HandLandmarksToRoi",
deps = [
"//mediapipe/calculators/core:split_normalized_landmark_list_calculator",
"//mediapipe/calculators/core:split_landmarks_calculator",
"//mediapipe/calculators/util:rect_transformation_calculator",
"//mediapipe/modules/hand_landmark/calculators:hand_landmarks_to_rect_calculator",
],
@@ -191,7 +191,7 @@ mediapipe_simple_subgraph(
deps = [
"//mediapipe/calculators/core:constant_side_packet_calculator",
"//mediapipe/calculators/core:side_packet_to_stream_calculator",
"//mediapipe/calculators/core:split_normalized_landmark_list_calculator",
"//mediapipe/calculators/core:split_landmarks_calculator",
"//mediapipe/calculators/util:set_landmark_visibility_calculator",
],
)
@@ -202,7 +202,7 @@ mediapipe_simple_subgraph(
register_as = "HandLandmarksLeftAndRightGpu",
deps = [
":hand_landmarks_from_pose_gpu",
"//mediapipe/calculators/core:split_normalized_landmark_list_calculator",
"//mediapipe/calculators/core:split_landmarks_calculator",
],
)
@@ -212,7 +212,7 @@ mediapipe_simple_subgraph(
register_as = "HandLandmarksLeftAndRightCpu",
deps = [
":hand_landmarks_from_pose_cpu",
"//mediapipe/calculators/core:split_normalized_landmark_list_calculator",
"//mediapipe/calculators/core:split_landmarks_calculator",
],
)
@@ -232,7 +232,7 @@ mediapipe_simple_subgraph(
graph = "hand_visibility_from_hand_landmarks_from_pose.pbtxt",
register_as = "HandVisibilityFromHandLandmarksFromPose",
deps = [
"//mediapipe/calculators/core:split_normalized_landmark_list_calculator",
"//mediapipe/calculators/core:split_landmarks_calculator",
"//mediapipe/calculators/util:landmark_visibility_calculator",
"//mediapipe/calculators/util:thresholding_calculator",
],
mediapipe/modules/holistic_landmark/face_detection_front_detections_to_roi.pbtxt
+1 -1
View File
@@ -1,4 +1,4 @@
# Calculates ROI from detections provided by `face_detection_front.tflite`
# Calculates ROI from detections provided by `face_detection_short_range.tflite`
# model.
type: "FaceDetectionFrontDetectionsToRoi"
mediapipe/modules/holistic_landmark/face_landmarks_from_pose_cpu.pbtxt
+2 -2
View File
@@ -43,7 +43,7 @@ node {
# to refine face ROI for further landmark detection as ROI calculated from
# pose landmarks may be inaccurate.
node {
calculator: "FaceDetectionFrontByRoiCpu"
calculator: "FaceDetectionShortRangeByRoiCpu"
input_stream: "IMAGE:input_video"
input_stream: "ROI:face_roi_from_pose"
output_stream: "DETECTIONS:face_detections"
@@ -51,7 +51,7 @@ node {
# Calculates refined face ROI.
node {
calculator: "FaceDetectionFrontDetectionsToRoi",
calculator: "FaceDetectionFrontDetectionsToRoi"
input_stream: "DETECTIONS:face_detections"
input_stream: "IMAGE_SIZE:image_size"
output_stream: "ROI:face_roi_from_detection"
mediapipe/modules/holistic_landmark/face_landmarks_from_pose_gpu.pbtxt
+2 -2
View File
@@ -43,7 +43,7 @@ node {
# to refine face ROI for further landmark detection as ROI calculated from
# pose landmarks may be inaccurate.
node {
calculator: "FaceDetectionFrontByRoiGpu"
calculator: "FaceDetectionShortRangeByRoiGpu"
input_stream: "IMAGE:input_video"
input_stream: "ROI:face_roi_from_pose"
output_stream: "DETECTIONS:face_detections"
@@ -51,7 +51,7 @@ node {
# Calculates refined face ROI.
node {
calculator: "FaceDetectionFrontDetectionsToRoi",
calculator: "FaceDetectionFrontDetectionsToRoi"
input_stream: "DETECTIONS:face_detections"
input_stream: "IMAGE_SIZE:image_size"
output_stream: "ROI:face_roi_from_detection"
mediapipe/modules/holistic_landmark/holistic_landmark_cpu.pbtxt
+5 -2
View File
@@ -1,8 +1,8 @@
# Predicts pose + left/right hand + face landmarks.
#
# It is required that:
# - "face_detection_front.tflite" is available at
# "mediapipe/modules/face_detection/face_detection_front.tflite"
# - "face_detection_short_range.tflite" is available at
# "mediapipe/modules/face_detection/face_detection_short_range.tflite"
#
# - "face_landmark.tflite" is available at
# "mediapipe/modules/face_landmark/face_landmark.tflite"
@@ -62,6 +62,8 @@ input_side_packet: "SMOOTH_LANDMARKS:smooth_landmarks"
# Pose landmarks. (NormalizedLandmarkList)
# 33 pose landmarks.
output_stream: "POSE_LANDMARKS:pose_landmarks"
# 33 pose world landmarks. (LandmarkList)
output_stream: "WORLD_LANDMARKS:pose_world_landmarks"
# 21 left hand landmarks. (NormalizedLandmarkList)
output_stream: "LEFT_HAND_LANDMARKS:left_hand_landmarks"
# 21 right hand landmarks. (NormalizedLandmarkList)
@@ -80,6 +82,7 @@ node {
input_side_packet: "MODEL_COMPLEXITY:model_complexity"
input_side_packet: "SMOOTH_LANDMARKS:smooth_landmarks"
output_stream: "LANDMARKS:pose_landmarks"
output_stream: "WORLD_LANDMARKS:pose_world_landmarks"
output_stream: "ROI_FROM_LANDMARKS:pose_landmarks_roi"
output_stream: "DETECTION:pose_detection"
}
mediapipe/modules/holistic_landmark/holistic_landmark_gpu.pbtxt
+5 -2
View File
@@ -1,8 +1,8 @@
# Predicts pose + left/right hand + face landmarks.
#
# It is required that:
# - "face_detection_front.tflite" is available at
# "mediapipe/modules/face_detection/face_detection_front.tflite"
# - "face_detection_short_range.tflite" is available at
# "mediapipe/modules/face_detection/face_detection_short_range.tflite"
#
# - "face_landmark.tflite" is available at
# "mediapipe/modules/face_landmark/face_landmark.tflite"
@@ -62,6 +62,8 @@ input_side_packet: "SMOOTH_LANDMARKS:smooth_landmarks"
# Pose landmarks. (NormalizedLandmarkList)
# 33 pose landmarks.
output_stream: "POSE_LANDMARKS:pose_landmarks"
# 33 pose world landmarks. (LandmarkList)
output_stream: "WORLD_LANDMARKS:pose_world_landmarks"
# 21 left hand landmarks. (NormalizedLandmarkList)
output_stream: "LEFT_HAND_LANDMARKS:left_hand_landmarks"
# 21 right hand landmarks. (NormalizedLandmarkList)
@@ -80,6 +82,7 @@ node {
input_side_packet: "MODEL_COMPLEXITY:model_complexity"
input_side_packet: "SMOOTH_LANDMARKS:smooth_landmarks"
output_stream: "LANDMARKS:pose_landmarks"
output_stream: "WORLD_LANDMARKS:pose_world_landmarks"
output_stream: "ROI_FROM_LANDMARKS:pose_landmarks_roi"
output_stream: "DETECTION:pose_detection"
}
mediapipe/modules/pose_detection/pose_detection.tflite
BIN
View File
Binary file not shown.
mediapipe/modules/pose_landmark/BUILD
+15 -12
View File
@@ -38,18 +38,10 @@ mediapipe_simple_subgraph(
graph = "pose_landmark_by_roi_gpu.pbtxt",
register_as = "PoseLandmarkByRoiGpu",
deps = [
":pose_landmark_by_roi_post_processing",
":pose_landmark_model_loader",
"//mediapipe/calculators/core:gate_calculator",
"//mediapipe/calculators/core:split_normalized_landmark_list_calculator",
"//mediapipe/calculators/core:split_vector_calculator",
"//mediapipe/calculators/tensor:image_to_tensor_calculator",
"//mediapipe/calculators/tensor:inference_calculator",
"//mediapipe/calculators/tensor:tensors_to_floats_calculator",
"//mediapipe/calculators/tensor:tensors_to_landmarks_calculator",
"//mediapipe/calculators/util:landmark_letterbox_removal_calculator",
"//mediapipe/calculators/util:landmark_projection_calculator",
"//mediapipe/calculators/util:refine_landmarks_from_heatmap_calculator",
"//mediapipe/calculators/util:thresholding_calculator",
],
)
@@ -58,18 +50,29 @@ mediapipe_simple_subgraph(
graph = "pose_landmark_by_roi_cpu.pbtxt",
register_as = "PoseLandmarkByRoiCpu",
deps = [
":pose_landmark_by_roi_post_processing",
":pose_landmark_model_loader",
"//mediapipe/calculators/core:gate_calculator",
"//mediapipe/calculators/core:split_normalized_landmark_list_calculator",
"//mediapipe/calculators/core:split_vector_calculator",
"//mediapipe/calculators/tensor:image_to_tensor_calculator",
"//mediapipe/calculators/tensor:inference_calculator",
],
)
mediapipe_simple_subgraph(
name = "pose_landmark_by_roi_post_processing",
graph = "pose_landmark_by_roi_post_processing.pbtxt",
register_as = "PoseLandmarkByRoiPostProcessing",
deps = [
"//mediapipe/calculators/core:gate_calculator",
"//mediapipe/calculators/core:split_landmarks_calculator",
"//mediapipe/calculators/core:split_vector_calculator",
"//mediapipe/calculators/tensor:tensors_to_floats_calculator",
"//mediapipe/calculators/tensor:tensors_to_landmarks_calculator",
"//mediapipe/calculators/util:landmark_letterbox_removal_calculator",
"//mediapipe/calculators/util:landmark_projection_calculator",
"//mediapipe/calculators/util:refine_landmarks_from_heatmap_calculator",
"//mediapipe/calculators/util:thresholding_calculator",
"//mediapipe/calculators/util:visibility_copy_calculator",
"//mediapipe/calculators/util:world_landmark_projection_calculator",
],
)
mediapipe/modules/pose_landmark/pose_landmark_by_roi_cpu.pbtxt
+14 -107
View File
@@ -77,6 +77,14 @@ output_stream: "LANDMARKS:landmarks"
# (NormalizedLandmarkList)
output_stream: "AUXILIARY_LANDMARKS:auxiliary_landmarks"
# Pose world landmarks within the given ROI. (LandmarkList)
# World landmarks are real-world 3D coordinates in meters with the origin at the
# center between hips. WORLD_LANDMARKS shares the same landmark topology as
# LANDMARKS. However, LANDMARKS provides coordinates (in pixels) of a 3D object
# projected onto the 2D image surface, while WORLD_LANDMARKS provides
# coordinates (in meters) of the 3D object itself.
output_stream: "WORLD_LANDMARKS:pose_world_landmarks"
# Transforms the input image into a 256x256 tensor while keeping the aspect
# ratio (what is expected by the corresponding model), resulting in potential
# letterboxing in the transformed image.
@@ -119,113 +127,12 @@ node {
}
}
# Splits a vector of TFLite tensors to multiple vectors according to the ranges
# specified in option.
node {
calculator: "SplitTensorVectorCalculator"
input_stream: "output_tensors"
output_stream: "landmark_tensors"
output_stream: "pose_flag_tensor"
output_stream: "heatmap_tensor"
options: {
[mediapipe.SplitVectorCalculatorOptions.ext] {
ranges: { begin: 0 end: 1 }
ranges: { begin: 1 end: 2 }
ranges: { begin: 3 end: 4 }
}
}
}
# Converts the pose-flag tensor into a float that represents the confidence
# score of pose presence.
node {
calculator: "TensorsToFloatsCalculator"
input_stream: "TENSORS:pose_flag_tensor"
output_stream: "FLOAT:pose_presence_score"
}
# Applies a threshold to the confidence score to determine whether a pose is
# present.
node {
calculator: "ThresholdingCalculator"
input_stream: "FLOAT:pose_presence_score"
output_stream: "FLAG:pose_presence"
options: {
[mediapipe.ThresholdingCalculatorOptions.ext] {
threshold: 0.5
}
}
}
# Drops landmark tensors if pose is not present.
node {
calculator: "GateCalculator"
input_stream: "landmark_tensors"
input_stream: "ALLOW:pose_presence"
output_stream: "ensured_landmark_tensors"
}
# Decodes the landmark tensors into a vector of landmarks, where the landmark
# coordinates are normalized by the size of the input image to the model.
node {
calculator: "TensorsToLandmarksCalculator"
input_stream: "TENSORS:ensured_landmark_tensors"
output_stream: "NORM_LANDMARKS:raw_landmarks"
options: {
[mediapipe.TensorsToLandmarksCalculatorOptions.ext] {
num_landmarks: 39
input_image_width: 256
input_image_height: 256
visibility_activation: SIGMOID
presence_activation: SIGMOID
}
}
}
# Refines landmarks with the heatmap tensor.
node {
calculator: "RefineLandmarksFromHeatmapCalculator"
input_stream: "NORM_LANDMARKS:raw_landmarks"
input_stream: "TENSORS:heatmap_tensor"
output_stream: "NORM_LANDMARKS:refined_landmarks"
options: {
[mediapipe.RefineLandmarksFromHeatmapCalculatorOptions.ext] {
kernel_size: 7
}
}
}
# Adjusts landmarks (already normalized to [0.f, 1.f]) on the letterboxed pose
# image (after image transformation with the FIT scale mode) to the
# corresponding locations on the same image with the letterbox removed (pose
# image before image transformation).
node {
calculator: "LandmarkLetterboxRemovalCalculator"
input_stream: "LANDMARKS:refined_landmarks"
calculator: "PoseLandmarkByRoiPostProcessing"
input_stream: "TENSORS:output_tensors"
input_stream: "ROI:roi"
input_stream: "LETTERBOX_PADDING:letterbox_padding"
output_stream: "LANDMARKS:adjusted_landmarks"
}
# Projects the landmarks from the cropped pose image to the corresponding
# locations on the full image before cropping (input to the graph).
node {
calculator: "LandmarkProjectionCalculator"
input_stream: "NORM_LANDMARKS:adjusted_landmarks"
input_stream: "NORM_RECT:roi"
output_stream: "NORM_LANDMARKS:all_landmarks"
}
# Splits the landmarks into two sets: the actual pose landmarks and the
# auxiliary landmarks.
node {
calculator: "SplitNormalizedLandmarkListCalculator"
input_stream: "all_landmarks"
output_stream: "landmarks"
output_stream: "auxiliary_landmarks"
options: {
[mediapipe.SplitVectorCalculatorOptions.ext] {
ranges: { begin: 0 end: 33 }
ranges: { begin: 33 end: 35 }
}
}
output_stream: "LANDMARKS:landmarks"
output_stream: "AUXILIARY_LANDMARKS:auxiliary_landmarks"
output_stream: "WORLD_LANDMARKS:pose_world_landmarks"
}
mediapipe/modules/pose_landmark/pose_landmark_by_roi_gpu.pbtxt
+14 -116
View File
@@ -77,6 +77,14 @@ output_stream: "LANDMARKS:landmarks"
# (NormalizedLandmarkList)
output_stream: "AUXILIARY_LANDMARKS:auxiliary_landmarks"
# Pose world landmarks within the given ROI. (LandmarkList)
# World landmarks are real-world 3D coordinates in meters with the origin at the
# center between hips. WORLD_LANDMARKS shares the same landmark topology as
# LANDMARKS. However, LANDMARKS provides coordinates (in pixels) of a 3D object
# projected onto the 2D image surface, while WORLD_LANDMARKS provides
# coordinates (in meters) of the 3D object itself.
output_stream: "WORLD_LANDMARKS:pose_world_landmarks"
# Transforms the input image into a 256x256 tensor while keeping the aspect
# ratio (what is expected by the corresponding model), resulting in potential
# letterboxing in the transformed image.
@@ -113,124 +121,14 @@ node {
input_side_packet: "MODEL:model"
input_stream: "TENSORS:input_tensors"
output_stream: "TENSORS:output_tensors"
options: {
[mediapipe.InferenceCalculatorOptions.ext] {
delegate {
gpu {
allow_precision_loss: false
}
}
}
}
}
# Splits a vector of TFLite tensors to multiple vectors according to the ranges
# specified in option.
node {
calculator: "SplitTensorVectorCalculator"
input_stream: "output_tensors"
output_stream: "landmark_tensors"
output_stream: "pose_flag_tensor"
output_stream: "heatmap_tensor"
options: {
[mediapipe.SplitVectorCalculatorOptions.ext] {
ranges: { begin: 0 end: 1 }
ranges: { begin: 1 end: 2 }
ranges: { begin: 3 end: 4 }
}
}
}
# Converts the pose-flag tensor into a float that represents the confidence
# score of pose presence.
node {
calculator: "TensorsToFloatsCalculator"
input_stream: "TENSORS:pose_flag_tensor"
output_stream: "FLOAT:pose_presence_score"
}
# Applies a threshold to the confidence score to determine whether a pose is
# present.
node {
calculator: "ThresholdingCalculator"
input_stream: "FLOAT:pose_presence_score"
output_stream: "FLAG:pose_presence"
options: {
[mediapipe.ThresholdingCalculatorOptions.ext] {
threshold: 0.5
}
}
}
# Drops landmark tensors if pose is not present.
node {
calculator: "GateCalculator"
input_stream: "landmark_tensors"
input_stream: "ALLOW:pose_presence"
output_stream: "ensured_landmark_tensors"
}
# Decodes the landmark tensors into a vector of landmarks, where the landmark
# coordinates are normalized by the size of the input image to the model.
node {
calculator: "TensorsToLandmarksCalculator"
input_stream: "TENSORS:ensured_landmark_tensors"
output_stream: "NORM_LANDMARKS:raw_landmarks"
options: {
[mediapipe.TensorsToLandmarksCalculatorOptions.ext] {
num_landmarks: 39
input_image_width: 256
input_image_height: 256
visibility_activation: SIGMOID
presence_activation: SIGMOID
}
}
}
# Refines landmarks with the heatmap tensor.
node {
calculator: "RefineLandmarksFromHeatmapCalculator"
input_stream: "NORM_LANDMARKS:raw_landmarks"
input_stream: "TENSORS:heatmap_tensor"
output_stream: "NORM_LANDMARKS:refined_landmarks"
options: {
[mediapipe.RefineLandmarksFromHeatmapCalculatorOptions.ext] {
kernel_size: 7
}
}
}
# Adjusts landmarks (already normalized to [0.f, 1.f]) on the letterboxed pose
# image (after image transformation with the FIT scale mode) to the
# corresponding locations on the same image with the letterbox removed (pose
# image before image transformation).
node {
calculator: "LandmarkLetterboxRemovalCalculator"
input_stream: "LANDMARKS:refined_landmarks"
calculator: "PoseLandmarkByRoiPostProcessing"
input_stream: "TENSORS:output_tensors"
input_stream: "ROI:roi"
input_stream: "LETTERBOX_PADDING:letterbox_padding"
output_stream: "LANDMARKS:adjusted_landmarks"
}
# Projects the landmarks from the cropped pose image to the corresponding
# locations on the full image before cropping (input to the graph).
node {
calculator: "LandmarkProjectionCalculator"
input_stream: "NORM_LANDMARKS:adjusted_landmarks"
input_stream: "NORM_RECT:roi"
output_stream: "NORM_LANDMARKS:all_landmarks"
}
# Splits the landmarks into two sets: the actual pose landmarks and the
# auxiliary landmarks.
node {
calculator: "SplitNormalizedLandmarkListCalculator"
input_stream: "all_landmarks"
output_stream: "landmarks"
output_stream: "auxiliary_landmarks"
options: {
[mediapipe.SplitVectorCalculatorOptions.ext] {
ranges: { begin: 0 end: 33 }
ranges: { begin: 33 end: 35 }
}
}
output_stream: "LANDMARKS:landmarks"
output_stream: "AUXILIARY_LANDMARKS:auxiliary_landmarks"
output_stream: "WORLD_LANDMARKS:pose_world_landmarks"
}
mediapipe/modules/pose_landmark/pose_landmark_by_roi_post_processing.pbtxt
+246
View File
@@ -0,0 +1,246 @@
# MediaPipe graph performing tensor post processing to detect/predict pose
# landmarks.
#
# EXAMPLE:
# node {
# calculator: "PoseLandmarkByRoiPostProcessing"
# input_stream: "TENSORS:tensors"
# input_stream: "ROI:roi"
# input_stream: "LETTERBOX_PADDING:letterbox_padding"
# output_stream: "LANDMARKS:landmarks"
# output_stream: "AUXILIARY_LANDMARKS:auxiliary_landmarks"
# }
type: "PoseLandmarkByRoiPostProcessing"
# Tensors from model inference of pose_landmark_{full|heavy|lite}.tflite.
# (std::vector<Tensor>)
input_stream: "TENSORS:tensors"
# ROI (region of interest) within a given image, where only the ROI is used as
# input to model inference but the resulting pose landmarks are to be reported
# in coordinates w.r.t. the entire image. (NormalizedRect)
input_stream: "ROI:roi"
# An array representing the letterbox padding from the 4 sides, i.e.,
# [left, top, right, bottom], padded around the ROI above before it is used as
# input to model inference. The padding is normalized to [0.f, 1.f] relatvie to
# the ROI dimensions. (std::array<float, 4>)
input_stream: "LETTERBOX_PADDING:letterbox_padding"
# Pose landmarks within the given ROI. (NormalizedLandmarkList)
# We have 33 landmarks (see pose_landmark_topology.svg) and there are other
# auxiliary key points.
# 0 - nose
# 1 - left eye (inner)
# 2 - left eye
# 3 - left eye (outer)
# 4 - right eye (inner)
# 5 - right eye
# 6 - right eye (outer)
# 7 - left ear
# 8 - right ear
# 9 - mouth (left)
# 10 - mouth (right)
# 11 - left shoulder
# 12 - right shoulder
# 13 - left elbow
# 14 - right elbow
# 15 - left wrist
# 16 - right wrist
# 17 - left pinky
# 18 - right pinky
# 19 - left index
# 20 - right index
# 21 - left thumb
# 22 - right thumb
# 23 - left hip
# 24 - right hip
# 25 - left knee
# 26 - right knee
# 27 - left ankle
# 28 - right ankle
# 29 - left heel
# 30 - right heel
# 31 - left foot index
# 32 - right foot index
#
# NOTE: if a pose is not present within the given ROI, for this particular
# timestamp there will not be an output packet in the LANDMARKS stream. However,
# the MediaPipe framework will internally inform the downstream calculators of
# the absence of this packet so that they don't wait for it unnecessarily.
output_stream: "LANDMARKS:landmarks"
# Auxiliary landmarks for deriving the ROI in the subsequent image.
# (NormalizedLandmarkList)
output_stream: "AUXILIARY_LANDMARKS:auxiliary_landmarks"
# Pose world landmarks within the given ROI. (LandmarkList)
# World landmarks are real-world 3D coordinates in meters with the origin at the
# center between hips. WORLD_LANDMARKS shares the same landmark topology as
# LANDMARKS. However, LANDMARKS provides coordinates (in pixels) of a 3D object
# projected onto the 2D image surface, while WORLD_LANDMARKS provides
# coordinates (in meters) of the 3D object itself.
output_stream: "WORLD_LANDMARKS:world_landmarks"
# Splits a vector of TFLite tensors to multiple vectors according to the ranges
# specified in option.
node {
calculator: "SplitTensorVectorCalculator"
input_stream: "tensors"
output_stream: "landmark_tensors"
output_stream: "pose_flag_tensor"
output_stream: "heatmap_tensor"
output_stream: "world_landmark_tensors"
options: {
[mediapipe.SplitVectorCalculatorOptions.ext] {
ranges: { begin: 0 end: 1 }
ranges: { begin: 1 end: 2 }
ranges: { begin: 3 end: 4 }
ranges: { begin: 4 end: 5 }
}
}
}
# Converts the pose-flag tensor into a float that represents the confidence
# score of pose presence.
node {
calculator: "TensorsToFloatsCalculator"
input_stream: "TENSORS:pose_flag_tensor"
output_stream: "FLOAT:pose_presence_score"
}
# Applies a threshold to the confidence score to determine whether a pose is
# present.
node {
calculator: "ThresholdingCalculator"
input_stream: "FLOAT:pose_presence_score"
output_stream: "FLAG:pose_presence"
options: {
[mediapipe.ThresholdingCalculatorOptions.ext] {
threshold: 0.5
}
}
}
# Drops landmark tensors if pose is not present.
node {
calculator: "GateCalculator"
input_stream: "landmark_tensors"
input_stream: "world_landmark_tensors"
input_stream: "ALLOW:pose_presence"
output_stream: "ensured_landmark_tensors"
output_stream: "ensured_world_landmark_tensors"
}
# Decodes the landmark tensors into a vector of landmarks, where the landmark
# coordinates are normalized by the size of the input image to the model.
node {
calculator: "TensorsToLandmarksCalculator"
input_stream: "TENSORS:ensured_landmark_tensors"
output_stream: "NORM_LANDMARKS:raw_landmarks"
options: {
[mediapipe.TensorsToLandmarksCalculatorOptions.ext] {
num_landmarks: 39
input_image_width: 256
input_image_height: 256
visibility_activation: SIGMOID
presence_activation: SIGMOID
}
}
}
# Refines landmarks with the heatmap tensor.
node {
calculator: "RefineLandmarksFromHeatmapCalculator"
input_stream: "NORM_LANDMARKS:raw_landmarks"
input_stream: "TENSORS:heatmap_tensor"
output_stream: "NORM_LANDMARKS:refined_landmarks"
options: {
[mediapipe.RefineLandmarksFromHeatmapCalculatorOptions.ext] {
kernel_size: 7
}
}
}
# Adjusts landmarks (already normalized to [0.f, 1.f]) on the letterboxed pose
# image (after image transformation with the FIT scale mode) to the
# corresponding locations on the same image with the letterbox removed (pose
# image before image transformation).
node {
calculator: "LandmarkLetterboxRemovalCalculator"
input_stream: "LANDMARKS:refined_landmarks"
input_stream: "LETTERBOX_PADDING:letterbox_padding"
output_stream: "LANDMARKS:adjusted_landmarks"
}
# Projects the landmarks from the cropped pose image to the corresponding
# locations on the full image before cropping.
node {
calculator: "LandmarkProjectionCalculator"
input_stream: "NORM_LANDMARKS:adjusted_landmarks"
input_stream: "NORM_RECT:roi"
output_stream: "NORM_LANDMARKS:all_landmarks"
}
# Splits the landmarks into two sets: the actual pose landmarks and the
# auxiliary landmarks.
node {
calculator: "SplitNormalizedLandmarkListCalculator"
input_stream: "all_landmarks"
output_stream: "landmarks"
output_stream: "auxiliary_landmarks"
options: {
[mediapipe.SplitVectorCalculatorOptions.ext] {
ranges: { begin: 0 end: 33 }
ranges: { begin: 33 end: 35 }
}
}
}
# Decodes the world landmark tensors into a vector of world landmarks.
node {
calculator: "TensorsToLandmarksCalculator"
input_stream: "TENSORS:ensured_world_landmark_tensors"
output_stream: "LANDMARKS:raw_world_landmarks"
options: {
[mediapipe.TensorsToLandmarksCalculatorOptions.ext] {
num_landmarks: 39
}
}
}
# Reuses the visibility and presence field in pose landmarks for the world
# landmarks.
node {
calculator: "VisibilityCopyCalculator"
input_stream: "NORM_LANDMARKS_FROM:raw_landmarks"
input_stream: "LANDMARKS_TO:raw_world_landmarks"
output_stream: "LANDMARKS_TO:raw_world_landmarks_with_visibility"
options: {
[mediapipe.VisibilityCopyCalculatorOptions.ext] {
copy_visibility: true
copy_presence: true
}
}
}
# Projects the world landmarks from the cropped pose image to the corresponding
# locations on the full image before cropping.
node {
calculator: "WorldLandmarkProjectionCalculator"
input_stream: "LANDMARKS:raw_world_landmarks_with_visibility"
input_stream: "NORM_RECT:roi"
output_stream: "LANDMARKS:all_world_landmarks"
}
# Takes only actual world landmarks.
node {
calculator: "SplitLandmarkListCalculator"
input_stream: "all_world_landmarks"
output_stream: "world_landmarks"
options: {
[mediapipe.SplitVectorCalculatorOptions.ext] {
ranges: { begin: 0 end: 33 }
}
}
}
mediapipe/modules/pose_landmark/pose_landmark_cpu.pbtxt
+12 -1
View File
@@ -37,7 +37,7 @@ input_side_packet: "SMOOTH_LANDMARKS:smooth_landmarks"
# functions as set to 1. (int)
input_side_packet: "MODEL_COMPLEXITY:model_complexity"
# Pose landmarks within the given ROI. (NormalizedLandmarkList)
# Pose landmarks. (NormalizedLandmarkList)
# We have 33 landmarks (see pose_landmark_topology.svg), and there are other
# auxiliary key points.
# 0 - nose
@@ -80,6 +80,14 @@ input_side_packet: "MODEL_COMPLEXITY:model_complexity"
# the absence of this packet so that they don't wait for it unnecessarily.
output_stream: "LANDMARKS:pose_landmarks"
# Pose world landmarks. (LandmarkList)
# World landmarks are real-world 3D coordinates in meters with the origin at the
# center between hips. WORLD_LANDMARKS shares the same landmark topology as
# LANDMARKS. However, LANDMARKS provides coordinates (in pixels) of a 3D object
# projected onto the 2D image surface, while WORLD_LANDMARKS provides
# coordinates (in meters) of the 3D object itself.
output_stream: "WORLD_LANDMARKS:pose_world_landmarks"
# Extra outputs (for debugging, for instance).
# Detected poses. (Detection)
output_stream: "DETECTION:pose_detection"
@@ -185,6 +193,7 @@ node {
input_stream: "ROI:pose_rect"
output_stream: "LANDMARKS:unfiltered_pose_landmarks"
output_stream: "AUXILIARY_LANDMARKS:unfiltered_auxiliary_landmarks"
output_stream: "WORLD_LANDMARKS:unfiltered_world_landmarks"
}
# Smoothes landmarks to reduce jitter.
@@ -194,8 +203,10 @@ node {
input_stream: "IMAGE_SIZE:image_size"
input_stream: "NORM_LANDMARKS:unfiltered_pose_landmarks"
input_stream: "AUX_NORM_LANDMARKS:unfiltered_auxiliary_landmarks"
input_stream: "WORLD_LANDMARKS:unfiltered_world_landmarks"
output_stream: "FILTERED_NORM_LANDMARKS:pose_landmarks"
output_stream: "FILTERED_AUX_NORM_LANDMARKS:auxiliary_landmarks"
output_stream: "FILTERED_WORLD_LANDMARKS:pose_world_landmarks"
}
# Calculates region of interest based on the auxiliary landmarks, to be used in
mediapipe/modules/pose_landmark/pose_landmark_filtering.pbtxt
+85 -5
View File
@@ -8,8 +8,10 @@
# input_stream: "IMAGE_SIZE:image_size"
# input_stream: "NORM_LANDMARKS:landmarks"
# input_stream: "AUX_NORM_LANDMARKS:aux_landmarks"
# input_stream: "WORLD_LANDMARKS:world_landmarks"
# output_stream: "FILTERED_NORM_LANDMARKS:filtered_landmarks"
# output_stream: "FILTERED_AUX_NORM_LANDMARKS:filtered_aux_landmarks"
# output_stream: "FILTERED_WORLD_LANDMARKS:filtered_world_landmarks"
# }
type: "PoseLandmarkFiltering"
@@ -20,14 +22,18 @@ input_side_packet: "ENABLE:enable"
# Size of the image (width & height) where the landmarks are estimated from.
# (std::pair<int, int>)
input_stream: "IMAGE_SIZE:image_size"
# Normalized landmarks. (NormalizedRect)
# Normalized landmarks. (NormalizedLandmarkList)
input_stream: "NORM_LANDMARKS:landmarks"
# Auxiliary set of normalized landmarks. (NormalizedRect)
# Auxiliary set of normalized landmarks. (NormalizedLandmarkList)
input_stream: "AUX_NORM_LANDMARKS:aux_landmarks"
# Filtered normalized landmarks. (NormalizedRect)
# World landmarks. (LandmarkList)
input_stream: "WORLD_LANDMARKS:world_landmarks"
# Filtered normalized landmarks. (NormalizedLandmarkList)
output_stream: "FILTERED_NORM_LANDMARKS:filtered_landmarks"
# Filtered auxiliary set of normalized landmarks. (NormalizedRect)
# Filtered auxiliary set of normalized landmarks. (NormalizedLandmarkList)
output_stream: "FILTERED_AUX_NORM_LANDMARKS:filtered_aux_landmarks"
# Filtered world landmarks. (LandmarkList)
output_stream: "FILTERED_WORLD_LANDMARKS:filtered_world_landmarks"
# Converts landmarks to a detection that tightly encloses all landmarks.
node {
@@ -124,7 +130,81 @@ node {
}
}
# Smoothes pose landmark visibilities to reduce jitter.
# Smoothes world landmark visibilities to reduce jitter.
node {
calculator: "SwitchContainer"
input_side_packet: "ENABLE:enable"
input_stream: "LANDMARKS:world_landmarks"
output_stream: "FILTERED_LANDMARKS:filtered_world_visibility"
options: {
[mediapipe.SwitchContainerOptions.ext] {
enable: true
contained_node: {
calculator: "VisibilitySmoothingCalculator"
options: {
[mediapipe.VisibilitySmoothingCalculatorOptions.ext] {
no_filter: {}
}
}
}
contained_node: {
calculator: "VisibilitySmoothingCalculator"
options: {
[mediapipe.VisibilitySmoothingCalculatorOptions.ext] {
low_pass_filter {
alpha: 0.1
}
}
}
}
}
}
}
# Smoothes world landmark coordinates to reduce jitter.
node {
calculator: "SwitchContainer"
input_side_packet: "ENABLE:enable"
input_stream: "LANDMARKS:filtered_world_visibility"
output_stream: "FILTERED_LANDMARKS:filtered_world_landmarks"
options: {
[mediapipe.SwitchContainerOptions.ext] {
enable: true
contained_node: {
calculator: "LandmarksSmoothingCalculator"
options: {
[mediapipe.LandmarksSmoothingCalculatorOptions.ext] {
no_filter: {}
}
}
}
contained_node: {
calculator: "LandmarksSmoothingCalculator"
options: {
[mediapipe.LandmarksSmoothingCalculatorOptions.ext] {
one_euro_filter {
# Min cutoff 0.1 results into ~ 0.02 alpha in landmark EMA filter
# when landmark is static.
min_cutoff: 0.1
# Beta 40.0 in combintation with min_cutoff 0.1 results into ~0.8
# alpha in landmark EMA filter when landmark is moving fast.
beta: 40.0
# Derivative cutoff 1.0 results into ~0.17 alpha in landmark
# velocity EMA filter.
derivate_cutoff: 1.0
# As world landmdarks are predicted in real world 3D coordintates
# in meters (rather than in pixels of input image) prediction
# scale does not depend on the pose size in the image.
disable_value_scaling: true
}
}
}
}
}
}
}
# Smoothes auxiliary landmark visibilities to reduce jitter.
node {
calculator: "VisibilitySmoothingCalculator"
input_stream: "NORM_LANDMARKS:aux_landmarks"
mediapipe/modules/pose_landmark/pose_landmark_full.tflite
BIN
View File
Binary file not shown.
mediapipe/modules/pose_landmark/pose_landmark_gpu.pbtxt
+12 -1
View File
@@ -37,7 +37,7 @@ input_side_packet: "SMOOTH_LANDMARKS:smooth_landmarks"
# functions as set to 1. (int)
input_side_packet: "MODEL_COMPLEXITY:model_complexity"
# Pose landmarks within the given ROI. (NormalizedLandmarkList)
# Pose landmarks. (NormalizedLandmarkList)
# We have 33 landmarks (see pose_landmark_topology.svg), and there are other
# auxiliary key points.
# 0 - nose
@@ -80,6 +80,14 @@ input_side_packet: "MODEL_COMPLEXITY:model_complexity"
# the absence of this packet so that they don't wait for it unnecessarily.
output_stream: "LANDMARKS:pose_landmarks"
# Pose world landmarks. (LandmarkList)
# World landmarks are real-world 3D coordinates in meters with the origin at the
# center between hips. WORLD_LANDMARKS shares the same landmark topology as
# LANDMARKS. However, LANDMARKS provides coordinates (in pixels) of a 3D object
# projected onto the 2D image surface, while WORLD_LANDMARKS provides
# coordinates (in meters) of the 3D object itself.
output_stream: "WORLD_LANDMARKS:pose_world_landmarks"
# Extra outputs (for debugging, for instance).
# Detected poses. (Detection)
output_stream: "DETECTION:pose_detection"
@@ -185,6 +193,7 @@ node {
input_stream: "ROI:pose_rect"
output_stream: "LANDMARKS:unfiltered_pose_landmarks"
output_stream: "AUXILIARY_LANDMARKS:unfiltered_auxiliary_landmarks"
output_stream: "WORLD_LANDMARKS:unfiltered_world_landmarks"
}
# Smoothes landmarks to reduce jitter.
@@ -194,8 +203,10 @@ node {
input_stream: "IMAGE_SIZE:image_size"
input_stream: "NORM_LANDMARKS:unfiltered_pose_landmarks"
input_stream: "AUX_NORM_LANDMARKS:unfiltered_auxiliary_landmarks"
input_stream: "WORLD_LANDMARKS:unfiltered_world_landmarks"
output_stream: "FILTERED_NORM_LANDMARKS:pose_landmarks"
output_stream: "FILTERED_AUX_NORM_LANDMARKS:auxiliary_landmarks"
output_stream: "FILTERED_WORLD_LANDMARKS:pose_world_landmarks"
}
# Calculates region of interest based on the auxiliary landmarks, to be used in
mediapipe/modules/pose_landmark/pose_landmark_heavy.tflite
BIN
View File
Binary file not shown.
mediapipe/modules/pose_landmark/pose_landmark_lite.tflite
BIN
View File
Binary file not shown.