Project import generated by Copybara.
GitOrigin-RevId: f7d09ed033907b893638a8eb4148efa11c0f09a6
This commit is contained in:
MediaPipe Team
@@ -7,7 +7,10 @@ Each module (represented as a subfolder) provides subgraphs and corresponding re
|
||||
| Module | Description |
|
||||
| :--- | :--- |
|
||||
| [`face_detection`](face_detection/README.md) | Subgraphs to detect faces. |
|
||||
| [`face_geometry`](face_geometry/README.md) | Subgraphs to extract face geometry. |
|
||||
| [`face_landmark`](face_landmark/README.md) | Subgraphs to detect and track face landmarks. |
|
||||
| [`hand_landmark`](hand_landmark/README.md) | Subgraphs to detect and track hand landmarks. |
|
||||
| [`iris_landmark`](iris_landmark/README.md) | Subgraphs to detect iris landmarks. |
|
||||
| [`palm_detection`](palm_detection/README.md) | Subgraphs to detect palms/hands. |
|
||||
| [`pose_detection`](pose_detection/README.md) | Subgraphs to detect poses. |
|
||||
| [`pose_landmark`](pose_landmark/README.md) | Subgraphs to detect and track pose landmarks. |
|
||||
|
||||
@@ -26,11 +26,10 @@ mediapipe_simple_subgraph(
|
||||
graph = "face_detection_front_cpu.pbtxt",
|
||||
register_as = "FaceDetectionFrontCpu",
|
||||
deps = [
|
||||
"//mediapipe/calculators/image:image_transformation_calculator",
|
||||
"//mediapipe/calculators/tensor:image_to_tensor_calculator",
|
||||
"//mediapipe/calculators/tensor:inference_calculator",
|
||||
"//mediapipe/calculators/tensor:tensors_to_detections_calculator",
|
||||
"//mediapipe/calculators/tflite:ssd_anchors_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_converter_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_inference_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_tensors_to_detections_calculator",
|
||||
"//mediapipe/calculators/util:detection_letterbox_removal_calculator",
|
||||
"//mediapipe/calculators/util:non_max_suppression_calculator",
|
||||
],
|
||||
@@ -41,11 +40,10 @@ mediapipe_simple_subgraph(
|
||||
graph = "face_detection_front_gpu.pbtxt",
|
||||
register_as = "FaceDetectionFrontGpu",
|
||||
deps = [
|
||||
"//mediapipe/calculators/image:image_transformation_calculator",
|
||||
"//mediapipe/calculators/tensor:image_to_tensor_calculator",
|
||||
"//mediapipe/calculators/tensor:inference_calculator",
|
||||
"//mediapipe/calculators/tensor:tensors_to_detections_calculator",
|
||||
"//mediapipe/calculators/tflite:ssd_anchors_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_converter_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_inference_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_tensors_to_detections_calculator",
|
||||
"//mediapipe/calculators/util:detection_letterbox_removal_calculator",
|
||||
"//mediapipe/calculators/util:non_max_suppression_calculator",
|
||||
],
|
||||
|
||||
@@ -24,41 +24,36 @@ input_stream: "IMAGE:image"
|
||||
# this packet so that they don't wait for it unnecessarily.
|
||||
output_stream: "DETECTIONS:detections"
|
||||
|
||||
# Transforms the input image on CPU to a 128x128 image. To scale the input
|
||||
# image, the scale_mode option is set to FIT to preserve the aspect ratio
|
||||
# (what is expected by the corresponding face detection model), resulting in
|
||||
# potential letterboxing in the transformed image.
|
||||
# Transforms the input image into a 128x128 tensor while keeping the aspect
|
||||
# ratio (what is expected by the corresponding face detection model), resulting
|
||||
# in potential letterboxing in the transformed image.
|
||||
node: {
|
||||
calculator: "ImageTransformationCalculator"
|
||||
calculator: "ImageToTensorCalculator"
|
||||
input_stream: "IMAGE:image"
|
||||
output_stream: "IMAGE:transformed_image"
|
||||
output_stream: "TENSORS:input_tensors"
|
||||
output_stream: "LETTERBOX_PADDING:letterbox_padding"
|
||||
options: {
|
||||
[mediapipe.ImageTransformationCalculatorOptions.ext] {
|
||||
output_width: 128
|
||||
output_height: 128
|
||||
scale_mode: FIT
|
||||
[mediapipe.ImageToTensorCalculatorOptions.ext] {
|
||||
output_tensor_width: 128
|
||||
output_tensor_height: 128
|
||||
keep_aspect_ratio: true
|
||||
output_tensor_float_range {
|
||||
min: -1.0
|
||||
max: 1.0
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Converts the transformed input image on CPU into an image tensor stored as a
|
||||
# TfLiteTensor.
|
||||
node {
|
||||
calculator: "TfLiteConverterCalculator"
|
||||
input_stream: "IMAGE:transformed_image"
|
||||
output_stream: "TENSORS:input_tensors"
|
||||
}
|
||||
|
||||
# Runs a TensorFlow Lite model on CPU that takes an image tensor and outputs a
|
||||
# vector of tensors representing, for instance, detection boxes/keypoints and
|
||||
# scores.
|
||||
node {
|
||||
calculator: "TfLiteInferenceCalculator"
|
||||
calculator: "InferenceCalculator"
|
||||
input_stream: "TENSORS:input_tensors"
|
||||
output_stream: "TENSORS:detection_tensors"
|
||||
options: {
|
||||
[mediapipe.TfLiteInferenceCalculatorOptions.ext] {
|
||||
[mediapipe.InferenceCalculatorOptions.ext] {
|
||||
model_path: "mediapipe/modules/face_detection/face_detection_front.tflite"
|
||||
delegate { xnnpack {} }
|
||||
}
|
||||
@@ -93,12 +88,12 @@ node {
|
||||
# the SSD anchors and the specification in the options, into a vector of
|
||||
# detections. Each detection describes a detected object.
|
||||
node {
|
||||
calculator: "TfLiteTensorsToDetectionsCalculator"
|
||||
calculator: "TensorsToDetectionsCalculator"
|
||||
input_stream: "TENSORS:detection_tensors"
|
||||
input_side_packet: "ANCHORS:anchors"
|
||||
output_stream: "DETECTIONS:unfiltered_detections"
|
||||
options: {
|
||||
[mediapipe.TfLiteTensorsToDetectionsCalculatorOptions.ext] {
|
||||
[mediapipe.TensorsToDetectionsCalculatorOptions.ext] {
|
||||
num_classes: 1
|
||||
num_boxes: 896
|
||||
num_coords: 16
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# MediaPipe graph to detect faces. (GPU input, and inference is executed on
|
||||
# GPU.)
|
||||
# MediaPipe graph to detect faces. (CPU input, and inference is executed on
|
||||
# CPU.)
|
||||
#
|
||||
# It is required that "face_detection_front.tflite" is available at
|
||||
# "mediapipe/modules/face_detection/face_detection_front.tflite"
|
||||
@@ -24,41 +24,37 @@ input_stream: "IMAGE:image"
|
||||
# this packet so that they don't wait for it unnecessarily.
|
||||
output_stream: "DETECTIONS:detections"
|
||||
|
||||
# Transforms the input image on GPU to a 128x128 image. To scale the input
|
||||
# image, the scale_mode option is set to FIT to preserve the aspect ratio
|
||||
# (what is expected by the corresponding face detection model), resulting in
|
||||
# potential letterboxing in the transformed image.
|
||||
# Transforms the input image into a 128x128 tensor while keeping the aspect
|
||||
# ratio (what is expected by the corresponding face detection model), resulting
|
||||
# in potential letterboxing in the transformed image.
|
||||
node: {
|
||||
calculator: "ImageTransformationCalculator"
|
||||
calculator: "ImageToTensorCalculator"
|
||||
input_stream: "IMAGE_GPU:image"
|
||||
output_stream: "IMAGE_GPU:transformed_image"
|
||||
output_stream: "TENSORS:input_tensors"
|
||||
output_stream: "LETTERBOX_PADDING:letterbox_padding"
|
||||
options: {
|
||||
[mediapipe.ImageTransformationCalculatorOptions.ext] {
|
||||
output_width: 128
|
||||
output_height: 128
|
||||
scale_mode: FIT
|
||||
[mediapipe.ImageToTensorCalculatorOptions.ext] {
|
||||
output_tensor_width: 128
|
||||
output_tensor_height: 128
|
||||
keep_aspect_ratio: true
|
||||
output_tensor_float_range {
|
||||
min: -1.0
|
||||
max: 1.0
|
||||
}
|
||||
gpu_origin: TOP_LEFT
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Converts the transformed input image on GPU into an image tensor stored as a
|
||||
# TfLiteTensor.
|
||||
node {
|
||||
calculator: "TfLiteConverterCalculator"
|
||||
input_stream: "IMAGE_GPU:transformed_image"
|
||||
output_stream: "TENSORS_GPU:input_tensors"
|
||||
}
|
||||
|
||||
# Runs a TensorFlow Lite model on GPU that takes an image tensor and outputs a
|
||||
# vector of tensors representing, for instance, detection boxes/keypoints and
|
||||
# scores.
|
||||
node {
|
||||
calculator: "TfLiteInferenceCalculator"
|
||||
input_stream: "TENSORS_GPU:input_tensors"
|
||||
output_stream: "TENSORS_GPU:detection_tensors"
|
||||
calculator: "InferenceCalculator"
|
||||
input_stream: "TENSORS:input_tensors"
|
||||
output_stream: "TENSORS:detection_tensors"
|
||||
options: {
|
||||
[mediapipe.TfLiteInferenceCalculatorOptions.ext] {
|
||||
[mediapipe.InferenceCalculatorOptions.ext] {
|
||||
model_path: "mediapipe/modules/face_detection/face_detection_front.tflite"
|
||||
}
|
||||
}
|
||||
@@ -92,12 +88,12 @@ node {
|
||||
# the SSD anchors and the specification in the options, into a vector of
|
||||
# detections. Each detection describes a detected object.
|
||||
node {
|
||||
calculator: "TfLiteTensorsToDetectionsCalculator"
|
||||
input_stream: "TENSORS_GPU:detection_tensors"
|
||||
calculator: "TensorsToDetectionsCalculator"
|
||||
input_stream: "TENSORS:detection_tensors"
|
||||
input_side_packet: "ANCHORS:anchors"
|
||||
output_stream: "DETECTIONS:unfiltered_detections"
|
||||
options: {
|
||||
[mediapipe.TfLiteTensorsToDetectionsCalculatorOptions.ext] {
|
||||
[mediapipe.TensorsToDetectionsCalculatorOptions.ext] {
|
||||
num_classes: 1
|
||||
num_boxes: 896
|
||||
num_coords: 16
|
||||
|
||||
@@ -28,12 +28,10 @@ mediapipe_simple_subgraph(
|
||||
deps = [
|
||||
"//mediapipe/calculators/core:gate_calculator",
|
||||
"//mediapipe/calculators/core:split_vector_calculator",
|
||||
"//mediapipe/calculators/image:image_cropping_calculator",
|
||||
"//mediapipe/calculators/image:image_transformation_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_converter_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_inference_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_tensors_to_floats_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_tensors_to_landmarks_calculator",
|
||||
"//mediapipe/calculators/tensor:image_to_tensor_calculator",
|
||||
"//mediapipe/calculators/tensor:inference_calculator",
|
||||
"//mediapipe/calculators/tensor:tensors_to_floats_calculator",
|
||||
"//mediapipe/calculators/tensor:tensors_to_landmarks_calculator",
|
||||
"//mediapipe/calculators/util:landmark_projection_calculator",
|
||||
"//mediapipe/calculators/util:thresholding_calculator",
|
||||
],
|
||||
@@ -46,12 +44,10 @@ mediapipe_simple_subgraph(
|
||||
deps = [
|
||||
"//mediapipe/calculators/core:gate_calculator",
|
||||
"//mediapipe/calculators/core:split_vector_calculator",
|
||||
"//mediapipe/calculators/image:image_cropping_calculator",
|
||||
"//mediapipe/calculators/image:image_transformation_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_converter_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_inference_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_tensors_to_floats_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_tensors_to_landmarks_calculator",
|
||||
"//mediapipe/calculators/tensor:image_to_tensor_calculator",
|
||||
"//mediapipe/calculators/tensor:inference_calculator",
|
||||
"//mediapipe/calculators/tensor:tensors_to_floats_calculator",
|
||||
"//mediapipe/calculators/tensor:tensors_to_landmarks_calculator",
|
||||
"//mediapipe/calculators/util:landmark_projection_calculator",
|
||||
"//mediapipe/calculators/util:thresholding_calculator",
|
||||
],
|
||||
@@ -74,6 +70,7 @@ mediapipe_simple_subgraph(
|
||||
"//mediapipe/calculators/image:image_properties_calculator",
|
||||
"//mediapipe/calculators/util:association_norm_rect_calculator",
|
||||
"//mediapipe/calculators/util:collection_has_min_size_calculator",
|
||||
"//mediapipe/calculators/util:logic_calculator",
|
||||
"//mediapipe/modules/face_detection:face_detection_front_cpu",
|
||||
],
|
||||
)
|
||||
@@ -95,6 +92,7 @@ mediapipe_simple_subgraph(
|
||||
"//mediapipe/calculators/image:image_properties_calculator",
|
||||
"//mediapipe/calculators/util:association_norm_rect_calculator",
|
||||
"//mediapipe/calculators/util:collection_has_min_size_calculator",
|
||||
"//mediapipe/calculators/util:logic_calculator",
|
||||
"//mediapipe/modules/face_detection:face_detection_front_gpu",
|
||||
],
|
||||
)
|
||||
|
||||
@@ -28,51 +28,33 @@ input_stream: "ROI:roi"
|
||||
# the absence of this packet so that they don't wait for it unnecessarily.
|
||||
output_stream: "LANDMARKS:face_landmarks"
|
||||
|
||||
# Crops the input image to the region of interest.
|
||||
node {
|
||||
calculator: "ImageCroppingCalculator"
|
||||
# Transforms the input image into a 192x192 tensor.
|
||||
node: {
|
||||
calculator: "ImageToTensorCalculator"
|
||||
input_stream: "IMAGE:image"
|
||||
input_stream: "NORM_RECT:roi"
|
||||
output_stream: "IMAGE:face_region"
|
||||
output_stream: "TENSORS:input_tensors"
|
||||
options: {
|
||||
[mediapipe.ImageCroppingCalculatorOptions.ext] {
|
||||
border_mode: BORDER_REPLICATE
|
||||
[mediapipe.ImageToTensorCalculatorOptions.ext] {
|
||||
output_tensor_width: 192
|
||||
output_tensor_height: 192
|
||||
output_tensor_float_range {
|
||||
min: 0.0
|
||||
max: 1.0
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Transforms the input image on CPU to a 192x192 image. To scale the input
|
||||
# image, the scale_mode option is set to FIT to preserve the aspect ratio,
|
||||
# resulting in potential letterboxing in the transformed image.
|
||||
node: {
|
||||
calculator: "ImageTransformationCalculator"
|
||||
input_stream: "IMAGE:face_region"
|
||||
output_stream: "IMAGE:transformed_face_region"
|
||||
options: {
|
||||
[mediapipe.ImageTransformationCalculatorOptions.ext] {
|
||||
output_width: 192
|
||||
output_height: 192
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Converts the transformed input image on CPU into an image tensor stored as a
|
||||
# TfLiteTensor.
|
||||
node {
|
||||
calculator: "TfLiteConverterCalculator"
|
||||
input_stream: "IMAGE:transformed_face_region"
|
||||
output_stream: "TENSORS:input_tensor"
|
||||
}
|
||||
|
||||
# Runs a TensorFlow Lite model on CPU that takes an image tensor and outputs a
|
||||
# vector of tensors representing, for instance, detection boxes/keypoints and
|
||||
# scores.
|
||||
node {
|
||||
calculator: "TfLiteInferenceCalculator"
|
||||
input_stream: "TENSORS:input_tensor"
|
||||
calculator: "InferenceCalculator"
|
||||
input_stream: "TENSORS:input_tensors"
|
||||
output_stream: "TENSORS:output_tensors"
|
||||
options: {
|
||||
[mediapipe.TfLiteInferenceCalculatorOptions.ext] {
|
||||
[mediapipe.InferenceCalculatorOptions.ext] {
|
||||
model_path: "mediapipe/modules/face_landmark/face_landmark.tflite"
|
||||
delegate { xnnpack {} }
|
||||
}
|
||||
@@ -81,7 +63,7 @@ node {
|
||||
|
||||
# Splits a vector of tensors into multiple vectors.
|
||||
node {
|
||||
calculator: "SplitTfLiteTensorVectorCalculator"
|
||||
calculator: "SplitTensorVectorCalculator"
|
||||
input_stream: "output_tensors"
|
||||
output_stream: "landmark_tensors"
|
||||
output_stream: "face_flag_tensor"
|
||||
@@ -96,7 +78,7 @@ node {
|
||||
# Converts the face-flag tensor into a float that represents the confidence
|
||||
# score of face presence.
|
||||
node {
|
||||
calculator: "TfLiteTensorsToFloatsCalculator"
|
||||
calculator: "TensorsToFloatsCalculator"
|
||||
input_stream: "TENSORS:face_flag_tensor"
|
||||
output_stream: "FLOAT:face_presence_score"
|
||||
}
|
||||
@@ -125,11 +107,11 @@ node {
|
||||
# Decodes the landmark tensors into a vector of lanmarks, where the landmark
|
||||
# coordinates are normalized by the size of the input image to the model.
|
||||
node {
|
||||
calculator: "TfLiteTensorsToLandmarksCalculator"
|
||||
calculator: "TensorsToLandmarksCalculator"
|
||||
input_stream: "TENSORS:ensured_landmark_tensors"
|
||||
output_stream: "NORM_LANDMARKS:landmarks"
|
||||
options: {
|
||||
[mediapipe.TfLiteTensorsToLandmarksCalculatorOptions.ext] {
|
||||
[mediapipe.TensorsToLandmarksCalculatorOptions.ext] {
|
||||
num_landmarks: 468
|
||||
input_image_width: 192
|
||||
input_image_height: 192
|
||||
|
||||
@@ -26,6 +26,10 @@ input_stream: "IMAGE:image"
|
||||
# Max number of faces to detect/track. (int)
|
||||
input_side_packet: "NUM_FACES:num_faces"
|
||||
|
||||
# Whether face detection can be skipped when face regions can already be
|
||||
# approximated from face landmarks on the previous frame.
|
||||
input_side_packet: "CAN_SKIP_DETECTION:can_skip_detection"
|
||||
|
||||
# Collection of detected/predicted faces, each represented as a list of 468 face
|
||||
# landmarks. (std::vector<NormalizedLandmarkList>)
|
||||
# NOTE: there will not be an output packet in the LANDMARKS stream for this
|
||||
@@ -54,12 +58,22 @@ node {
|
||||
}
|
||||
|
||||
# Drops the incoming image if FaceLandmarkCpu was able to identify face presence
|
||||
# in the previous image. Otherwise, passes the incoming image through to trigger
|
||||
# a new round of face detection in FaceDetectionFrontCpu.
|
||||
# in the previous image and skipping face detection is enabled. Otherwise,
|
||||
# passes the incoming image through to trigger a new round of face detection
|
||||
# in FaceDetectionFrontCpu.
|
||||
node {
|
||||
calculator: "LogicCalculator"
|
||||
options: {
|
||||
[mediapipe.LogicCalculatorOptions.ext] { op: AND }
|
||||
}
|
||||
input_side_packet: "can_skip_detection"
|
||||
input_stream: "prev_has_enough_faces"
|
||||
output_stream: "skip_detection"
|
||||
}
|
||||
node {
|
||||
calculator: "GateCalculator"
|
||||
input_stream: "image"
|
||||
input_stream: "DISALLOW:prev_has_enough_faces"
|
||||
input_stream: "DISALLOW:skip_detection"
|
||||
output_stream: "gated_image"
|
||||
options: {
|
||||
[mediapipe.GateCalculatorOptions.ext] {
|
||||
@@ -67,6 +81,12 @@ node {
|
||||
}
|
||||
}
|
||||
}
|
||||
node {
|
||||
calculator: "GateCalculator"
|
||||
input_stream: "prev_face_rects_from_landmarks"
|
||||
input_stream: "ALLOW:skip_detection"
|
||||
output_stream: "gated_prev_face_rects_from_landmarks"
|
||||
}
|
||||
|
||||
# Detects faces.
|
||||
node {
|
||||
@@ -129,7 +149,7 @@ node {
|
||||
# overlapping regions based on the specified min_similarity_threshold.
|
||||
node {
|
||||
calculator: "AssociationNormRectCalculator"
|
||||
input_stream: "prev_face_rects_from_landmarks"
|
||||
input_stream: "gated_prev_face_rects_from_landmarks"
|
||||
input_stream: "face_rects_from_detections"
|
||||
output_stream: "face_rects"
|
||||
options: {
|
||||
|
||||
@@ -26,6 +26,10 @@ input_stream: "IMAGE:image"
|
||||
# Max number of faces to detect/track. (int)
|
||||
input_side_packet: "NUM_FACES:num_faces"
|
||||
|
||||
# Whether face detection can be skipped when face regions can already be
|
||||
# approximated from face landmarks on the previous frame.
|
||||
input_side_packet: "CAN_SKIP_DETECTION:can_skip_detection"
|
||||
|
||||
# Collection of detected/predicted faces, each represented as a list of 468 face
|
||||
# landmarks. (std::vector<NormalizedLandmarkList>)
|
||||
# NOTE: there will not be an output packet in the LANDMARKS stream for this
|
||||
@@ -54,12 +58,22 @@ node {
|
||||
}
|
||||
|
||||
# Drops the incoming image if FaceLandmarkGpu was able to identify face presence
|
||||
# in the previous image. Otherwise, passes the incoming image through to trigger
|
||||
# a new round of face detection in FaceDetectionFrontGpu.
|
||||
# in the previous image and skipping face detection is enabled. Otherwise,
|
||||
# passes the incoming image through to trigger a new round of face detection
|
||||
# in FaceDetectionFrontGpu.
|
||||
node {
|
||||
calculator: "LogicCalculator"
|
||||
options: {
|
||||
[mediapipe.LogicCalculatorOptions.ext] { op: AND }
|
||||
}
|
||||
input_side_packet: "can_skip_detection"
|
||||
input_stream: "prev_has_enough_faces"
|
||||
output_stream: "skip_detection"
|
||||
}
|
||||
node {
|
||||
calculator: "GateCalculator"
|
||||
input_stream: "image"
|
||||
input_stream: "DISALLOW:prev_has_enough_faces"
|
||||
input_stream: "DISALLOW:skip_detection"
|
||||
output_stream: "gated_image"
|
||||
options: {
|
||||
[mediapipe.GateCalculatorOptions.ext] {
|
||||
@@ -67,6 +81,12 @@ node {
|
||||
}
|
||||
}
|
||||
}
|
||||
node {
|
||||
calculator: "GateCalculator"
|
||||
input_stream: "prev_face_rects_from_landmarks"
|
||||
input_stream: "ALLOW:skip_detection"
|
||||
output_stream: "gated_prev_face_rects_from_landmarks"
|
||||
}
|
||||
|
||||
# Detects faces.
|
||||
node {
|
||||
@@ -129,7 +149,7 @@ node {
|
||||
# overlapping regions based on the specified min_similarity_threshold.
|
||||
node {
|
||||
calculator: "AssociationNormRectCalculator"
|
||||
input_stream: "prev_face_rects_from_landmarks"
|
||||
input_stream: "gated_prev_face_rects_from_landmarks"
|
||||
input_stream: "face_rects_from_detections"
|
||||
output_stream: "face_rects"
|
||||
options: {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# MediaPipe graph to detect/predict face landmarks. (GPU input, and inference is
|
||||
# executed on GPU.)
|
||||
# MediaPipe graph to detect/predict face landmarks. (CPU input, and inference is
|
||||
# executed on CPU.)
|
||||
#
|
||||
# It is required that "face_landmark.tflite" is available at
|
||||
# "mediapipe/modules/face_landmark/face_landmark.tflite"
|
||||
@@ -28,51 +28,34 @@ input_stream: "ROI:roi"
|
||||
# the absence of this packet so that they don't wait for it unnecessarily.
|
||||
output_stream: "LANDMARKS:face_landmarks"
|
||||
|
||||
# Crops the input image to the given region of interest.
|
||||
node {
|
||||
calculator: "ImageCroppingCalculator"
|
||||
# Transforms the input image into a 192x192 tensor.
|
||||
node: {
|
||||
calculator: "ImageToTensorCalculator"
|
||||
input_stream: "IMAGE_GPU:image"
|
||||
input_stream: "NORM_RECT:roi"
|
||||
output_stream: "IMAGE_GPU:face_region"
|
||||
output_stream: "TENSORS:input_tensors"
|
||||
options: {
|
||||
[mediapipe.ImageCroppingCalculatorOptions.ext] {
|
||||
border_mode: BORDER_REPLICATE
|
||||
[mediapipe.ImageToTensorCalculatorOptions.ext] {
|
||||
output_tensor_width: 192
|
||||
output_tensor_height: 192
|
||||
output_tensor_float_range {
|
||||
min: 0.0
|
||||
max: 1.0
|
||||
}
|
||||
gpu_origin: TOP_LEFT
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Transforms the input image on GPU to a 192x192 image. To scale the input
|
||||
# image, the scale_mode option is set to FIT to preserve the aspect ratio,
|
||||
# resulting in potential letterboxing in the transformed image.
|
||||
node: {
|
||||
calculator: "ImageTransformationCalculator"
|
||||
input_stream: "IMAGE_GPU:face_region"
|
||||
output_stream: "IMAGE_GPU:transformed_face_region"
|
||||
options: {
|
||||
[mediapipe.ImageTransformationCalculatorOptions.ext] {
|
||||
output_width: 192
|
||||
output_height: 192
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Converts the transformed input image on GPU into an image tensor stored as a
|
||||
# TfLiteTensor.
|
||||
node {
|
||||
calculator: "TfLiteConverterCalculator"
|
||||
input_stream: "IMAGE_GPU:transformed_face_region"
|
||||
output_stream: "TENSORS_GPU:input_tensor"
|
||||
}
|
||||
|
||||
# Runs a TensorFlow Lite model on GPU that takes an image tensor and outputs a
|
||||
# vector of GPU tensors representing, for instance, detection boxes/keypoints
|
||||
# and scores.
|
||||
node {
|
||||
calculator: "TfLiteInferenceCalculator"
|
||||
input_stream: "TENSORS_GPU:input_tensor"
|
||||
calculator: "InferenceCalculator"
|
||||
input_stream: "TENSORS:input_tensors"
|
||||
output_stream: "TENSORS:output_tensors"
|
||||
options: {
|
||||
[mediapipe.TfLiteInferenceCalculatorOptions.ext] {
|
||||
[mediapipe.InferenceCalculatorOptions.ext] {
|
||||
model_path: "mediapipe/modules/face_landmark/face_landmark.tflite"
|
||||
}
|
||||
}
|
||||
@@ -80,7 +63,7 @@ node {
|
||||
|
||||
# Splits a vector of tensors into multiple vectors.
|
||||
node {
|
||||
calculator: "SplitTfLiteTensorVectorCalculator"
|
||||
calculator: "SplitTensorVectorCalculator"
|
||||
input_stream: "output_tensors"
|
||||
output_stream: "landmark_tensors"
|
||||
output_stream: "face_flag_tensor"
|
||||
@@ -95,7 +78,7 @@ node {
|
||||
# Converts the face-flag tensor into a float that represents the confidence
|
||||
# score of face presence.
|
||||
node {
|
||||
calculator: "TfLiteTensorsToFloatsCalculator"
|
||||
calculator: "TensorsToFloatsCalculator"
|
||||
input_stream: "TENSORS:face_flag_tensor"
|
||||
output_stream: "FLOAT:face_presence_score"
|
||||
}
|
||||
@@ -124,11 +107,11 @@ node {
|
||||
# Decodes the landmark tensors into a vector of lanmarks, where the landmark
|
||||
# coordinates are normalized by the size of the input image to the model.
|
||||
node {
|
||||
calculator: "TfLiteTensorsToLandmarksCalculator"
|
||||
calculator: "TensorsToLandmarksCalculator"
|
||||
input_stream: "TENSORS:ensured_landmark_tensors"
|
||||
output_stream: "NORM_LANDMARKS:landmarks"
|
||||
options: {
|
||||
[mediapipe.TfLiteTensorsToLandmarksCalculatorOptions.ext] {
|
||||
[mediapipe.TensorsToLandmarksCalculatorOptions.ext] {
|
||||
num_landmarks: 468
|
||||
input_image_width: 192
|
||||
input_image_height: 192
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
# Copyright 2020 The MediaPipe Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
load(
|
||||
"//mediapipe/framework/tool:mediapipe_graph.bzl",
|
||||
"mediapipe_simple_subgraph",
|
||||
)
|
||||
|
||||
licenses(["notice"])
|
||||
|
||||
package(default_visibility = ["//visibility:public"])
|
||||
|
||||
exports_files([
|
||||
"hand_landmark.tflite",
|
||||
"handedness.txt",
|
||||
])
|
||||
|
||||
mediapipe_simple_subgraph(
|
||||
name = "hand_landmark_cpu",
|
||||
graph = "hand_landmark_cpu.pbtxt",
|
||||
register_as = "HandLandmarkCpu",
|
||||
deps = [
|
||||
"//mediapipe/calculators/core:gate_calculator",
|
||||
"//mediapipe/calculators/core:split_vector_calculator",
|
||||
"//mediapipe/calculators/tensor:image_to_tensor_calculator",
|
||||
"//mediapipe/calculators/tensor:inference_calculator",
|
||||
"//mediapipe/calculators/tensor:tensors_to_classification_calculator",
|
||||
"//mediapipe/calculators/tensor:tensors_to_floats_calculator",
|
||||
"//mediapipe/calculators/tensor:tensors_to_landmarks_calculator",
|
||||
"//mediapipe/calculators/util:landmark_letterbox_removal_calculator",
|
||||
"//mediapipe/calculators/util:landmark_projection_calculator",
|
||||
"//mediapipe/calculators/util:logic_calculator",
|
||||
"//mediapipe/calculators/util:thresholding_calculator",
|
||||
],
|
||||
)
|
||||
|
||||
mediapipe_simple_subgraph(
|
||||
name = "hand_landmark_gpu",
|
||||
graph = "hand_landmark_gpu.pbtxt",
|
||||
register_as = "HandLandmarkGpu",
|
||||
deps = [
|
||||
"//mediapipe/calculators/core:gate_calculator",
|
||||
"//mediapipe/calculators/core:split_vector_calculator",
|
||||
"//mediapipe/calculators/tensor:image_to_tensor_calculator",
|
||||
"//mediapipe/calculators/tensor:inference_calculator",
|
||||
"//mediapipe/calculators/tensor:tensors_to_classification_calculator",
|
||||
"//mediapipe/calculators/tensor:tensors_to_floats_calculator",
|
||||
"//mediapipe/calculators/tensor:tensors_to_landmarks_calculator",
|
||||
"//mediapipe/calculators/util:landmark_letterbox_removal_calculator",
|
||||
"//mediapipe/calculators/util:landmark_projection_calculator",
|
||||
"//mediapipe/calculators/util:logic_calculator",
|
||||
"//mediapipe/calculators/util:thresholding_calculator",
|
||||
],
|
||||
)
|
||||
|
||||
mediapipe_simple_subgraph(
|
||||
name = "hand_landmark_tracking_gpu",
|
||||
graph = "hand_landmark_tracking_gpu.pbtxt",
|
||||
register_as = "HandLandmarkTrackingGpu",
|
||||
deps = [
|
||||
":hand_landmark_gpu",
|
||||
":hand_landmark_landmarks_to_roi",
|
||||
":palm_detection_detection_to_roi",
|
||||
"//mediapipe/calculators/core:begin_loop_calculator",
|
||||
"//mediapipe/calculators/core:clip_vector_size_calculator",
|
||||
"//mediapipe/calculators/core:end_loop_calculator",
|
||||
"//mediapipe/calculators/core:flow_limiter_calculator",
|
||||
"//mediapipe/calculators/core:gate_calculator",
|
||||
"//mediapipe/calculators/core:previous_loopback_calculator",
|
||||
"//mediapipe/calculators/image:image_properties_calculator",
|
||||
"//mediapipe/calculators/util:association_norm_rect_calculator",
|
||||
"//mediapipe/calculators/util:collection_has_min_size_calculator",
|
||||
"//mediapipe/calculators/util:filter_collection_calculator",
|
||||
"//mediapipe/modules/palm_detection:palm_detection_gpu",
|
||||
],
|
||||
)
|
||||
|
||||
mediapipe_simple_subgraph(
|
||||
name = "hand_landmark_tracking_cpu",
|
||||
graph = "hand_landmark_tracking_cpu.pbtxt",
|
||||
register_as = "HandLandmarkTrackingCpu",
|
||||
deps = [
|
||||
":hand_landmark_cpu",
|
||||
":hand_landmark_landmarks_to_roi",
|
||||
":palm_detection_detection_to_roi",
|
||||
"//mediapipe/calculators/core:begin_loop_calculator",
|
||||
"//mediapipe/calculators/core:clip_vector_size_calculator",
|
||||
"//mediapipe/calculators/core:end_loop_calculator",
|
||||
"//mediapipe/calculators/core:flow_limiter_calculator",
|
||||
"//mediapipe/calculators/core:gate_calculator",
|
||||
"//mediapipe/calculators/core:previous_loopback_calculator",
|
||||
"//mediapipe/calculators/image:image_properties_calculator",
|
||||
"//mediapipe/calculators/util:association_norm_rect_calculator",
|
||||
"//mediapipe/calculators/util:collection_has_min_size_calculator",
|
||||
"//mediapipe/calculators/util:filter_collection_calculator",
|
||||
"//mediapipe/modules/palm_detection:palm_detection_gpu",
|
||||
],
|
||||
)
|
||||
|
||||
mediapipe_simple_subgraph(
|
||||
name = "palm_detection_detection_to_roi",
|
||||
graph = "palm_detection_detection_to_roi.pbtxt",
|
||||
register_as = "PalmDetectionDetectionToRoi",
|
||||
deps = [
|
||||
"//mediapipe/calculators/util:detections_to_rects_calculator",
|
||||
"//mediapipe/calculators/util:rect_transformation_calculator",
|
||||
],
|
||||
)
|
||||
|
||||
mediapipe_simple_subgraph(
|
||||
name = "hand_landmark_landmarks_to_roi",
|
||||
graph = "hand_landmark_landmarks_to_roi.pbtxt",
|
||||
register_as = "HandLandmarkLandmarksToRoi",
|
||||
deps = [
|
||||
"//mediapipe/calculators/core:split_normalized_landmark_list_calculator",
|
||||
"//mediapipe/calculators/util:rect_transformation_calculator",
|
||||
"//mediapipe/modules/hand_landmark/calculators:hand_landmarks_to_rect_calculator",
|
||||
],
|
||||
)
|
||||
@@ -0,0 +1,8 @@
|
||||
# hand_landmark
|
||||
|
||||
Subgraphs|Details
|
||||
:--- | :---
|
||||
[`HandLandmarkCpu`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/hand_landmark/hand_landmark_cpu.pbtxt)| Detects landmarks of a single hand. (CPU input.)
|
||||
[`HandLandmarkGpu`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/hand_landmark/hand_landmark_gpu.pbtxt)| Detects landmarks of a single hand. (GPU input.)
|
||||
[`HandLandmarkTrackingCpu`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/hand_landmark/hand_landmark_tracking_cpu.pbtxt)| Detects and tracks landmarks of multiple hands. (CPU input.)
|
||||
[`HandLandmarkTrackingGpu`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/hand_landmark/hand_landmark_tracking_gpu.pbtxt)| Detects and tracks landmarks of multiple hands. (GPU input.)
|
||||
@@ -0,0 +1,33 @@
|
||||
# Copyright 2020 The MediaPipe Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
licenses(["notice"])
|
||||
|
||||
package(default_visibility = ["//visibility:public"])
|
||||
|
||||
cc_library(
|
||||
name = "hand_landmarks_to_rect_calculator",
|
||||
srcs = ["hand_landmarks_to_rect_calculator.cc"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:calculator_options_cc_proto",
|
||||
"//mediapipe/framework/formats:landmark_cc_proto",
|
||||
"//mediapipe/framework/formats:location_data_cc_proto",
|
||||
"//mediapipe/framework/formats:rect_cc_proto",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/port:status",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
@@ -0,0 +1,167 @@
|
||||
// Copyright 2020 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#include <cmath>
|
||||
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/calculator_options.pb.h"
|
||||
#include "mediapipe/framework/formats/landmark.pb.h"
|
||||
#include "mediapipe/framework/formats/rect.pb.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
namespace {
|
||||
|
||||
constexpr char kNormalizedLandmarksTag[] = "NORM_LANDMARKS";
|
||||
constexpr char kNormRectTag[] = "NORM_RECT";
|
||||
constexpr char kImageSizeTag[] = "IMAGE_SIZE";
|
||||
constexpr int kWristJoint = 0;
|
||||
constexpr int kMiddleFingerPIPJoint = 6;
|
||||
constexpr int kIndexFingerPIPJoint = 4;
|
||||
constexpr int kRingFingerPIPJoint = 8;
|
||||
constexpr float kTargetAngle = M_PI * 0.5f;
|
||||
|
||||
inline float NormalizeRadians(float angle) {
|
||||
return angle - 2 * M_PI * std::floor((angle - (-M_PI)) / (2 * M_PI));
|
||||
}
|
||||
|
||||
float ComputeRotation(const NormalizedLandmarkList& landmarks,
|
||||
const std::pair<int, int>& image_size) {
|
||||
const float x0 = landmarks.landmark(kWristJoint).x() * image_size.first;
|
||||
const float y0 = landmarks.landmark(kWristJoint).y() * image_size.second;
|
||||
|
||||
float x1 = (landmarks.landmark(kIndexFingerPIPJoint).x() +
|
||||
landmarks.landmark(kRingFingerPIPJoint).x()) /
|
||||
2.f;
|
||||
float y1 = (landmarks.landmark(kIndexFingerPIPJoint).y() +
|
||||
landmarks.landmark(kRingFingerPIPJoint).y()) /
|
||||
2.f;
|
||||
x1 = (x1 + landmarks.landmark(kMiddleFingerPIPJoint).x()) / 2.f *
|
||||
image_size.first;
|
||||
y1 = (y1 + landmarks.landmark(kMiddleFingerPIPJoint).y()) / 2.f *
|
||||
image_size.second;
|
||||
|
||||
const float rotation =
|
||||
NormalizeRadians(kTargetAngle - std::atan2(-(y1 - y0), x1 - x0));
|
||||
return rotation;
|
||||
}
|
||||
|
||||
::mediapipe::Status NormalizedLandmarkListToRect(
|
||||
const NormalizedLandmarkList& landmarks,
|
||||
const std::pair<int, int>& image_size, NormalizedRect* rect) {
|
||||
const float rotation = ComputeRotation(landmarks, image_size);
|
||||
const float reverse_angle = NormalizeRadians(-rotation);
|
||||
|
||||
// Find boundaries of landmarks.
|
||||
float max_x = std::numeric_limits<float>::min();
|
||||
float max_y = std::numeric_limits<float>::min();
|
||||
float min_x = std::numeric_limits<float>::max();
|
||||
float min_y = std::numeric_limits<float>::max();
|
||||
for (int i = 0; i < landmarks.landmark_size(); ++i) {
|
||||
max_x = std::max(max_x, landmarks.landmark(i).x());
|
||||
max_y = std::max(max_y, landmarks.landmark(i).y());
|
||||
min_x = std::min(min_x, landmarks.landmark(i).x());
|
||||
min_y = std::min(min_y, landmarks.landmark(i).y());
|
||||
}
|
||||
const float axis_aligned_center_x = (max_x + min_x) / 2.f;
|
||||
const float axis_aligned_center_y = (max_y + min_y) / 2.f;
|
||||
|
||||
// Find boundaries of rotated landmarks.
|
||||
max_x = std::numeric_limits<float>::min();
|
||||
max_y = std::numeric_limits<float>::min();
|
||||
min_x = std::numeric_limits<float>::max();
|
||||
min_y = std::numeric_limits<float>::max();
|
||||
for (int i = 0; i < landmarks.landmark_size(); ++i) {
|
||||
const float original_x =
|
||||
(landmarks.landmark(i).x() - axis_aligned_center_x) * image_size.first;
|
||||
const float original_y =
|
||||
(landmarks.landmark(i).y() - axis_aligned_center_y) * image_size.second;
|
||||
|
||||
const float projected_x = original_x * std::cos(reverse_angle) -
|
||||
original_y * std::sin(reverse_angle);
|
||||
const float projected_y = original_x * std::sin(reverse_angle) +
|
||||
original_y * std::cos(reverse_angle);
|
||||
|
||||
max_x = std::max(max_x, projected_x);
|
||||
max_y = std::max(max_y, projected_y);
|
||||
min_x = std::min(min_x, projected_x);
|
||||
min_y = std::min(min_y, projected_y);
|
||||
}
|
||||
const float projected_center_x = (max_x + min_x) / 2.f;
|
||||
const float projected_center_y = (max_y + min_y) / 2.f;
|
||||
|
||||
const float center_x = projected_center_x * std::cos(rotation) -
|
||||
projected_center_y * std::sin(rotation) +
|
||||
image_size.first * axis_aligned_center_x;
|
||||
const float center_y = projected_center_x * std::sin(rotation) +
|
||||
projected_center_y * std::cos(rotation) +
|
||||
image_size.second * axis_aligned_center_y;
|
||||
const float width = (max_x - min_x) / image_size.first;
|
||||
const float height = (max_y - min_y) / image_size.second;
|
||||
|
||||
rect->set_x_center(center_x / image_size.first);
|
||||
rect->set_y_center(center_y / image_size.second);
|
||||
rect->set_width(width);
|
||||
rect->set_height(height);
|
||||
rect->set_rotation(rotation);
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
// A calculator that converts subset of hand landmarks to a bounding box
|
||||
// NormalizedRect. The rotation angle of the bounding box is computed based on
|
||||
// 1) the wrist joint and 2) the average of PIP joints of index finger, middle
|
||||
// finger and ring finger. After rotation, the vector from the wrist to the mean
|
||||
// of PIP joints is expected to be vertical with wrist at the bottom and the
|
||||
// mean of PIP joints at the top.
|
||||
class HandLandmarksToRectCalculator : public CalculatorBase {
|
||||
public:
|
||||
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
|
||||
cc->Inputs().Tag(kNormalizedLandmarksTag).Set<NormalizedLandmarkList>();
|
||||
cc->Inputs().Tag(kImageSizeTag).Set<std::pair<int, int>>();
|
||||
cc->Outputs().Tag(kNormRectTag).Set<NormalizedRect>();
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status Open(CalculatorContext* cc) override {
|
||||
cc->SetOffset(TimestampDiff(0));
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status Process(CalculatorContext* cc) override {
|
||||
if (cc->Inputs().Tag(kNormalizedLandmarksTag).IsEmpty()) {
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
RET_CHECK(!cc->Inputs().Tag(kImageSizeTag).IsEmpty());
|
||||
|
||||
std::pair<int, int> image_size =
|
||||
cc->Inputs().Tag(kImageSizeTag).Get<std::pair<int, int>>();
|
||||
const auto& landmarks =
|
||||
cc->Inputs().Tag(kNormalizedLandmarksTag).Get<NormalizedLandmarkList>();
|
||||
auto output_rect = absl::make_unique<NormalizedRect>();
|
||||
MP_RETURN_IF_ERROR(
|
||||
NormalizedLandmarkListToRect(landmarks, image_size, output_rect.get()));
|
||||
cc->Outputs()
|
||||
.Tag(kNormRectTag)
|
||||
.Add(output_rect.release(), cc->InputTimestamp());
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
};
|
||||
REGISTER_CALCULATOR(HandLandmarksToRectCalculator);
|
||||
|
||||
} // namespace mediapipe
|
||||
Binary file not shown.
@@ -0,0 +1,163 @@
|
||||
# MediaPipe graph to detect/predict hand landmarks on CPU.
|
||||
|
||||
type: "HandLandmarkCpu"
|
||||
|
||||
# CPU image. (ImageFrame)
|
||||
input_stream: "IMAGE:image"
|
||||
# ROI (region of interest) within the given image where a palm/hand is located.
|
||||
# (NormalizedRect)
|
||||
input_stream: "ROI:hand_rect"
|
||||
|
||||
# 21 hand landmarks within the given ROI. (NormalizedLandmarkList)
|
||||
# NOTE: if a hand is not present within the given ROI, for this particular
|
||||
# timestamp there will not be an output packet in the LANDMARKS stream. However,
|
||||
# the MediaPipe framework will internally inform the downstream calculators of
|
||||
# the absence of this packet so that they don't wait for it unnecessarily.
|
||||
output_stream: "LANDMARKS:hand_landmarks"
|
||||
|
||||
# Handedness of the detected hand (i.e. is hand left or right).
|
||||
# (ClassificationList)
|
||||
output_stream: "HANDEDNESS:handedness"
|
||||
|
||||
# Transforms a region of image into a 224x224 tensor while keeping the aspect
|
||||
# ratio, and therefore may result in potential letterboxing.
|
||||
node {
|
||||
calculator: "ImageToTensorCalculator"
|
||||
input_stream: "IMAGE:image"
|
||||
input_stream: "NORM_RECT:hand_rect"
|
||||
output_stream: "TENSORS:input_tensor"
|
||||
output_stream: "LETTERBOX_PADDING:letterbox_padding"
|
||||
options: {
|
||||
[mediapipe.ImageToTensorCalculatorOptions.ext] {
|
||||
output_tensor_width: 224
|
||||
output_tensor_height: 224
|
||||
keep_aspect_ratio: true
|
||||
output_tensor_float_range {
|
||||
min: 0.0
|
||||
max: 1.0
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Runs a TensorFlow Lite model on CPU that takes an image tensor and outputs a
|
||||
# vector of tensors representing, for instance, detection boxes/keypoints and
|
||||
# scores.
|
||||
node {
|
||||
calculator: "InferenceCalculator"
|
||||
input_stream: "TENSORS:input_tensor"
|
||||
output_stream: "TENSORS:output_tensors"
|
||||
options: {
|
||||
[mediapipe.InferenceCalculatorOptions.ext] {
|
||||
model_path: "mediapipe/modules/hand_landmark/hand_landmark.tflite"
|
||||
delegate { xnnpack {} }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Splits a vector of tensors to multiple vectors according to the ranges
|
||||
# specified in option.
|
||||
node {
|
||||
calculator: "SplitTensorVectorCalculator"
|
||||
input_stream: "output_tensors"
|
||||
output_stream: "landmark_tensors"
|
||||
output_stream: "hand_flag_tensor"
|
||||
output_stream: "handedness_tensor"
|
||||
options: {
|
||||
[mediapipe.SplitVectorCalculatorOptions.ext] {
|
||||
ranges: { begin: 0 end: 1 }
|
||||
ranges: { begin: 1 end: 2 }
|
||||
ranges: { begin: 2 end: 3 }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Converts the hand-flag tensor into a float that represents the confidence
|
||||
# score of hand presence.
|
||||
node {
|
||||
calculator: "TensorsToFloatsCalculator"
|
||||
input_stream: "TENSORS:hand_flag_tensor"
|
||||
output_stream: "FLOAT:hand_presence_score"
|
||||
}
|
||||
|
||||
# Applies a threshold to the confidence score to determine whether a hand is
|
||||
# present.
|
||||
node {
|
||||
calculator: "ThresholdingCalculator"
|
||||
input_stream: "FLOAT:hand_presence_score"
|
||||
output_stream: "FLAG:hand_presence"
|
||||
options: {
|
||||
[mediapipe.ThresholdingCalculatorOptions.ext] {
|
||||
threshold: 0.5
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Drops handedness tensor if hand is not present.
|
||||
node {
|
||||
calculator: "GateCalculator"
|
||||
input_stream: "handedness_tensor"
|
||||
input_stream: "ALLOW:hand_presence"
|
||||
output_stream: "ensured_handedness_tensor"
|
||||
}
|
||||
|
||||
# Converts the handedness tensor into a float that represents the classification
|
||||
# score of handedness.
|
||||
node {
|
||||
calculator: "TensorsToClassificationCalculator"
|
||||
input_stream: "TENSORS:ensured_handedness_tensor"
|
||||
output_stream: "CLASSIFICATIONS:handedness"
|
||||
options: {
|
||||
[mediapipe.TensorsToClassificationCalculatorOptions.ext] {
|
||||
top_k: 1
|
||||
label_map_path: "mediapipe/modules/hand_landmark/handedness.txt"
|
||||
binary_classification: true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Drops landmarks tensors if hand is not present.
|
||||
node {
|
||||
calculator: "GateCalculator"
|
||||
input_stream: "landmark_tensors"
|
||||
input_stream: "ALLOW:hand_presence"
|
||||
output_stream: "ensured_landmark_tensors"
|
||||
}
|
||||
|
||||
# Decodes the landmark tensors into a list of landmarks, where the landmark
|
||||
# coordinates are normalized by the size of the input image to the model.
|
||||
node {
|
||||
calculator: "TensorsToLandmarksCalculator"
|
||||
input_stream: "TENSORS:ensured_landmark_tensors"
|
||||
output_stream: "NORM_LANDMARKS:landmarks"
|
||||
options: {
|
||||
[mediapipe.TensorsToLandmarksCalculatorOptions.ext] {
|
||||
num_landmarks: 21
|
||||
input_image_width: 224
|
||||
input_image_height: 224
|
||||
# The additional scaling factor is used to account for the Z coordinate
|
||||
# distribution in the training data.
|
||||
normalize_z: 0.4
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Adjusts landmarks (already normalized to [0.f, 1.f]) on the letterboxed hand
|
||||
# image (after image transformation with the FIT scale mode) to the
|
||||
# corresponding locations on the same image with the letterbox removed (hand
|
||||
# image before image transformation).
|
||||
node {
|
||||
calculator: "LandmarkLetterboxRemovalCalculator"
|
||||
input_stream: "LANDMARKS:landmarks"
|
||||
input_stream: "LETTERBOX_PADDING:letterbox_padding"
|
||||
output_stream: "LANDMARKS:scaled_landmarks"
|
||||
}
|
||||
|
||||
# Projects the landmarks from the cropped hand image to the corresponding
|
||||
# locations on the full image before cropping (input to the graph).
|
||||
node {
|
||||
calculator: "LandmarkProjectionCalculator"
|
||||
input_stream: "NORM_LANDMARKS:scaled_landmarks"
|
||||
input_stream: "NORM_RECT:hand_rect"
|
||||
output_stream: "NORM_LANDMARKS:hand_landmarks"
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
# MediaPipe graph to detect/predict hand landmarks on CPU.
|
||||
|
||||
type: "HandLandmarkGpu"
|
||||
|
||||
# GPU image. (GpuBuffer)
|
||||
input_stream: "IMAGE:image"
|
||||
# ROI (region of interest) within the given image where a palm/hand is located.
|
||||
# (NormalizedRect)
|
||||
input_stream: "ROI:hand_rect"
|
||||
|
||||
# 21 hand landmarks within the given ROI. (NormalizedLandmarkList)
|
||||
# NOTE: if a hand is not present within the given ROI, for this particular
|
||||
# timestamp there will not be an output packet in the LANDMARKS stream. However,
|
||||
# the MediaPipe framework will internally inform the downstream calculators of
|
||||
# the absence of this packet so that they don't wait for it unnecessarily.
|
||||
output_stream: "LANDMARKS:hand_landmarks"
|
||||
|
||||
# Handedness of the detected hand (i.e. is hand left or right).
|
||||
# (ClassificationList)
|
||||
output_stream: "HANDEDNESS:handedness"
|
||||
|
||||
# Transforms a region of image into a 224x224 tensor while keeping the aspect
|
||||
# ratio, and therefore may result in potential letterboxing.
|
||||
node {
|
||||
calculator: "ImageToTensorCalculator"
|
||||
input_stream: "IMAGE_GPU:image"
|
||||
input_stream: "NORM_RECT:hand_rect"
|
||||
output_stream: "TENSORS:input_tensor"
|
||||
output_stream: "LETTERBOX_PADDING:letterbox_padding"
|
||||
options: {
|
||||
[mediapipe.ImageToTensorCalculatorOptions.ext] {
|
||||
output_tensor_width: 224
|
||||
output_tensor_height: 224
|
||||
keep_aspect_ratio: true
|
||||
output_tensor_float_range {
|
||||
min: 0.0
|
||||
max: 1.0
|
||||
}
|
||||
gpu_origin: TOP_LEFT
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Runs a TensorFlow Lite model on GPU that takes an image tensor and outputs a
|
||||
# vector of tensors representing, for instance, detection boxes/keypoints and
|
||||
# scores.
|
||||
node {
|
||||
calculator: "InferenceCalculator"
|
||||
input_stream: "TENSORS:input_tensor"
|
||||
output_stream: "TENSORS:output_tensors"
|
||||
options: {
|
||||
[mediapipe.InferenceCalculatorOptions.ext] {
|
||||
model_path: "mediapipe/modules/hand_landmark/hand_landmark.tflite"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Splits a vector of tensors to multiple vectors according to the ranges
|
||||
# specified in option.
|
||||
node {
|
||||
calculator: "SplitTensorVectorCalculator"
|
||||
input_stream: "output_tensors"
|
||||
output_stream: "landmark_tensors"
|
||||
output_stream: "hand_flag_tensor"
|
||||
output_stream: "handedness_tensor"
|
||||
options: {
|
||||
[mediapipe.SplitVectorCalculatorOptions.ext] {
|
||||
ranges: { begin: 0 end: 1 }
|
||||
ranges: { begin: 1 end: 2 }
|
||||
ranges: { begin: 2 end: 3 }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Converts the hand-flag tensor into a float that represents the confidence
|
||||
# score of hand presence.
|
||||
node {
|
||||
calculator: "TensorsToFloatsCalculator"
|
||||
input_stream: "TENSORS:hand_flag_tensor"
|
||||
output_stream: "FLOAT:hand_presence_score"
|
||||
}
|
||||
|
||||
# Applies a threshold to the confidence score to determine whether a hand is
|
||||
# present.
|
||||
node {
|
||||
calculator: "ThresholdingCalculator"
|
||||
input_stream: "FLOAT:hand_presence_score"
|
||||
output_stream: "FLAG:hand_presence"
|
||||
options: {
|
||||
[mediapipe.ThresholdingCalculatorOptions.ext] {
|
||||
threshold: 0.5
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Drops handedness tensor if hand is not present.
|
||||
node {
|
||||
calculator: "GateCalculator"
|
||||
input_stream: "handedness_tensor"
|
||||
input_stream: "ALLOW:hand_presence"
|
||||
output_stream: "ensured_handedness_tensor"
|
||||
}
|
||||
|
||||
# Converts the handedness tensor into a float that represents the classification
|
||||
# score of handedness.
|
||||
node {
|
||||
calculator: "TensorsToClassificationCalculator"
|
||||
input_stream: "TENSORS:ensured_handedness_tensor"
|
||||
output_stream: "CLASSIFICATIONS:handedness"
|
||||
options: {
|
||||
[mediapipe.TensorsToClassificationCalculatorOptions.ext] {
|
||||
top_k: 1
|
||||
label_map_path: "mediapipe/modules/hand_landmark/handedness.txt"
|
||||
binary_classification: true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Drops landmarks tensors if hand is not present.
|
||||
node {
|
||||
calculator: "GateCalculator"
|
||||
input_stream: "landmark_tensors"
|
||||
input_stream: "ALLOW:hand_presence"
|
||||
output_stream: "ensured_landmark_tensors"
|
||||
}
|
||||
|
||||
# Decodes the landmark tensors into a list of landmarks, where the landmark
|
||||
# coordinates are normalized by the size of the input image to the model.
|
||||
node {
|
||||
calculator: "TensorsToLandmarksCalculator"
|
||||
input_stream: "TENSORS:ensured_landmark_tensors"
|
||||
output_stream: "NORM_LANDMARKS:landmarks"
|
||||
options: {
|
||||
[mediapipe.TensorsToLandmarksCalculatorOptions.ext] {
|
||||
num_landmarks: 21
|
||||
input_image_width: 224
|
||||
input_image_height: 224
|
||||
# The additional scaling factor is used to account for the Z coordinate
|
||||
# distribution in the training data.
|
||||
normalize_z: 0.4
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Adjusts landmarks (already normalized to [0.f, 1.f]) on the letterboxed hand
|
||||
# image (after image transformation with the FIT scale mode) to the
|
||||
# corresponding locations on the same image with the letterbox removed (hand
|
||||
# image before image transformation).
|
||||
node {
|
||||
calculator: "LandmarkLetterboxRemovalCalculator"
|
||||
input_stream: "LANDMARKS:landmarks"
|
||||
input_stream: "LETTERBOX_PADDING:letterbox_padding"
|
||||
output_stream: "LANDMARKS:scaled_landmarks"
|
||||
}
|
||||
|
||||
# Projects the landmarks from the cropped hand image to the corresponding
|
||||
# locations on the full image before cropping (input to the graph).
|
||||
node {
|
||||
calculator: "LandmarkProjectionCalculator"
|
||||
input_stream: "NORM_LANDMARKS:scaled_landmarks"
|
||||
input_stream: "NORM_RECT:hand_rect"
|
||||
output_stream: "NORM_LANDMARKS:hand_landmarks"
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
# MediaPipe graph to calculate hand region of interest (ROI) from landmarks
|
||||
# detected by "HandLandmarkCpu" or "HandLandmarkGpu".
|
||||
|
||||
type: "HandLandmarkLandmarksToRoi"
|
||||
|
||||
# Normalized landmarks. (NormalizedLandmarkList)
|
||||
input_stream: "LANDMARKS:landmarks"
|
||||
# Image size (width & height). (std::pair<int, int>)
|
||||
input_stream: "IMAGE_SIZE:image_size"
|
||||
|
||||
# ROI according to landmarks. (NormalizedRect)
|
||||
output_stream: "ROI:roi"
|
||||
|
||||
# Extracts a subset of the hand landmarks that are relatively more stable across
|
||||
# frames (e.g. comparing to finger tips) for computing the bounding box. The box
|
||||
# will later be expanded to contain the entire hand. In this approach, it is
|
||||
# more robust to drastically changing hand size.
|
||||
# The landmarks extracted are: wrist, MCP/PIP of five fingers.
|
||||
node {
|
||||
calculator: "SplitNormalizedLandmarkListCalculator"
|
||||
input_stream: "landmarks"
|
||||
output_stream: "partial_landmarks"
|
||||
options: {
|
||||
[mediapipe.SplitVectorCalculatorOptions.ext] {
|
||||
ranges: { begin: 0 end: 4 }
|
||||
ranges: { begin: 5 end: 7 }
|
||||
ranges: { begin: 9 end: 11 }
|
||||
ranges: { begin: 13 end: 15 }
|
||||
ranges: { begin: 17 end: 19 }
|
||||
combine_outputs: true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Converts the hand landmarks into a rectangle (normalized by image size)
|
||||
# that encloses the hand. The calculator uses a subset of all hand landmarks
|
||||
# extracted from SplitNormalizedLandmarkListCalculator above to
|
||||
# calculate the bounding box and the rotation of the output rectangle. Please
|
||||
# see the comments in the calculator for more detail.
|
||||
node {
|
||||
calculator: "HandLandmarksToRectCalculator"
|
||||
input_stream: "NORM_LANDMARKS:partial_landmarks"
|
||||
input_stream: "IMAGE_SIZE:image_size"
|
||||
output_stream: "NORM_RECT:hand_rect_from_landmarks"
|
||||
}
|
||||
|
||||
# Expands the hand rectangle so that the box contains the entire hand and it's
|
||||
# big enough so that it's likely to still contain the hand even with some motion
|
||||
# in the next video frame .
|
||||
node {
|
||||
calculator: "RectTransformationCalculator"
|
||||
input_stream: "NORM_RECT:hand_rect_from_landmarks"
|
||||
input_stream: "IMAGE_SIZE:image_size"
|
||||
output_stream: "roi"
|
||||
options: {
|
||||
[mediapipe.RectTransformationCalculatorOptions.ext] {
|
||||
scale_x: 2.0
|
||||
scale_y: 2.0
|
||||
shift_y: -0.1
|
||||
square_long: true
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,241 @@
|
||||
# MediaPipe graph to detect/predict hand landmarks on CPU.
|
||||
#
|
||||
# The procedure is done in two steps:
|
||||
# - locate palms/hands
|
||||
# - detect landmarks for each palm/hand.
|
||||
# This graph tries to skip palm detection as much as possible by reusing
|
||||
# previously detected/predicted landmarks for new images.
|
||||
|
||||
type: "HandLandmarkTrackingCpu"
|
||||
|
||||
# CPU image. (ImageFrame)
|
||||
input_stream: "IMAGE:image"
|
||||
|
||||
# Max number of hands to detect/track. (int)
|
||||
input_side_packet: "NUM_HANDS:num_hands"
|
||||
|
||||
# Whether palm detection can be skipped when hand regions can already be
|
||||
# approximated from hand landmarks on the previous frame.
|
||||
input_side_packet: "CAN_SKIP_DETECTION:can_skip_detection"
|
||||
|
||||
# Collection of detected/predicted hands, each represented as a list of
|
||||
# landmarks. (std::vector<NormalizedLandmarkList>)
|
||||
# NOTE: there will not be an output packet in the LANDMARKS stream for this
|
||||
# particular timestamp if none of hands detected. However, the MediaPipe
|
||||
# framework will internally inform the downstream calculators of the absence of
|
||||
# this packet so that they don't wait for it unnecessarily.
|
||||
output_stream: "LANDMARKS:multi_hand_landmarks"
|
||||
# Collection of handedness of the detected hands (i.e. is hand left or right),
|
||||
# each represented as a Classification proto.
|
||||
# Note that handedness is determined assuming the input image is mirrored,
|
||||
# i.e., taken with a front-facing/selfie camera with images flipped
|
||||
# horizontally.
|
||||
output_stream: "HANDEDNESS:multi_handedness"
|
||||
|
||||
# Extra outputs (for debugging, for instance).
|
||||
# Detected palms. (std::vector<Detection>)
|
||||
output_stream: "PALM_DETECTIONS:palm_detections"
|
||||
# Regions of interest calculated based on landmarks.
|
||||
# (std::vector<NormalizedRect>)
|
||||
output_stream: "HAND_ROIS_FROM_LANDMARKS:hand_rects"
|
||||
# Regions of interest calculated based on palm detections.
|
||||
# (std::vector<NormalizedRect>)
|
||||
output_stream: "HAND_ROIS_FROM_PALM_DETECTIONS:hand_rects_from_palm_detections"
|
||||
|
||||
# Determines if an input vector of NormalizedRect has a size greater than or
|
||||
# equal to the provided num_hands.
|
||||
node {
|
||||
calculator: "NormalizedRectVectorHasMinSizeCalculator"
|
||||
input_stream: "ITERABLE:prev_hand_rects_from_landmarks"
|
||||
input_side_packet: "num_hands"
|
||||
output_stream: "prev_has_enough_hands"
|
||||
}
|
||||
|
||||
# Drops the incoming image if the previous image had at least N hands.
|
||||
# and skipping palm detection is enabled.
|
||||
# Otherwise, passes the incoming image through to trigger a new round of palm
|
||||
# detection.
|
||||
node {
|
||||
calculator: "LogicCalculator"
|
||||
options: {
|
||||
[mediapipe.LogicCalculatorOptions.ext] { op: AND }
|
||||
}
|
||||
input_side_packet: "can_skip_detection"
|
||||
input_stream: "prev_has_enough_hands"
|
||||
output_stream: "skip_detection"
|
||||
}
|
||||
|
||||
node {
|
||||
calculator: "GateCalculator"
|
||||
input_stream: "image"
|
||||
input_stream: "DISALLOW:skip_detection"
|
||||
output_stream: "palm_detection_image"
|
||||
options: {
|
||||
[mediapipe.GateCalculatorOptions.ext] {
|
||||
empty_packets_as_allow: true
|
||||
}
|
||||
}
|
||||
}
|
||||
node {
|
||||
calculator: "GateCalculator"
|
||||
input_stream: "prev_hand_rects_from_landmarks"
|
||||
input_stream: "ALLOW:skip_detection"
|
||||
output_stream: "gated_prev_hand_rects_from_landmarks"
|
||||
}
|
||||
|
||||
# Detects palms.
|
||||
node {
|
||||
calculator: "PalmDetectionCpu"
|
||||
input_stream: "IMAGE:palm_detection_image"
|
||||
output_stream: "DETECTIONS:all_palm_detections"
|
||||
}
|
||||
|
||||
# Makes sure there are no more detections than the provided num_hands.
|
||||
node {
|
||||
calculator: "ClipDetectionVectorSizeCalculator"
|
||||
input_stream: "all_palm_detections"
|
||||
output_stream: "palm_detections"
|
||||
input_side_packet: "num_hands"
|
||||
}
|
||||
|
||||
# Extracts image size.
|
||||
node {
|
||||
calculator: "ImagePropertiesCalculator"
|
||||
input_stream: "IMAGE:palm_detection_image"
|
||||
output_stream: "SIZE:palm_detection_image_size"
|
||||
}
|
||||
|
||||
# Outputs each element of palm_detections at a fake timestamp for the rest of
|
||||
# the graph to process. Clones the image size packet for each palm_detection at
|
||||
# the fake timestamp. At the end of the loop, outputs the BATCH_END timestamp
|
||||
# for downstream calculators to inform them that all elements in the vector have
|
||||
# been processed.
|
||||
node {
|
||||
calculator: "BeginLoopDetectionCalculator"
|
||||
input_stream: "ITERABLE:palm_detections"
|
||||
input_stream: "CLONE:palm_detection_image_size"
|
||||
output_stream: "ITEM:palm_detection"
|
||||
output_stream: "CLONE:image_size_for_palms"
|
||||
output_stream: "BATCH_END:palm_detections_timestamp"
|
||||
}
|
||||
|
||||
# Calculates region of interest (ROI) based on the specified palm.
|
||||
node {
|
||||
calculator: "PalmDetectionDetectionToRoi"
|
||||
input_stream: "DETECTION:palm_detection"
|
||||
input_stream: "IMAGE_SIZE:image_size_for_palms"
|
||||
output_stream: "ROI:hand_rect_from_palm_detection"
|
||||
}
|
||||
|
||||
# Collects a NormalizedRect for each hand into a vector. Upon receiving the
|
||||
# BATCH_END timestamp, outputs the vector of NormalizedRect at the BATCH_END
|
||||
# timestamp.
|
||||
node {
|
||||
calculator: "EndLoopNormalizedRectCalculator"
|
||||
input_stream: "ITEM:hand_rect_from_palm_detection"
|
||||
input_stream: "BATCH_END:palm_detections_timestamp"
|
||||
output_stream: "ITERABLE:hand_rects_from_palm_detections"
|
||||
}
|
||||
|
||||
# Performs association between NormalizedRect vector elements from previous
|
||||
# image and rects based on palm detections from the current image. This
|
||||
# calculator ensures that the output hand_rects vector doesn't contain
|
||||
# overlapping regions based on the specified min_similarity_threshold.
|
||||
node {
|
||||
calculator: "AssociationNormRectCalculator"
|
||||
input_stream: "gated_prev_hand_rects_from_landmarks"
|
||||
input_stream: "hand_rects_from_palm_detections"
|
||||
output_stream: "hand_rects"
|
||||
options: {
|
||||
[mediapipe.AssociationCalculatorOptions.ext] {
|
||||
min_similarity_threshold: 0.5
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Extracts image size.
|
||||
node {
|
||||
calculator: "ImagePropertiesCalculator"
|
||||
input_stream: "IMAGE:image"
|
||||
output_stream: "SIZE:image_size"
|
||||
}
|
||||
|
||||
# Outputs each element of hand_rects at a fake timestamp for the rest of the
|
||||
# graph to process. Clones image and image size packets for each
|
||||
# single_hand_rect at the fake timestamp. At the end of the loop, outputs the
|
||||
# BATCH_END timestamp for downstream calculators to inform them that all
|
||||
# elements in the vector have been processed.
|
||||
node {
|
||||
calculator: "BeginLoopNormalizedRectCalculator"
|
||||
input_stream: "ITERABLE:hand_rects"
|
||||
input_stream: "CLONE:0:image"
|
||||
input_stream: "CLONE:1:image_size"
|
||||
output_stream: "ITEM:single_hand_rect"
|
||||
output_stream: "CLONE:0:image_for_landmarks"
|
||||
output_stream: "CLONE:1:image_size_for_landmarks"
|
||||
output_stream: "BATCH_END:hand_rects_timestamp"
|
||||
}
|
||||
|
||||
# Detect hand landmarks for the specific hand rect.
|
||||
node {
|
||||
calculator: "HandLandmarkCpu"
|
||||
input_stream: "IMAGE:image_for_landmarks"
|
||||
input_stream: "ROI:single_hand_rect"
|
||||
output_stream: "LANDMARKS:single_hand_landmarks"
|
||||
output_stream: "HANDEDNESS:single_handedness"
|
||||
}
|
||||
|
||||
# Collects the handedness for each single hand into a vector. Upon
|
||||
# receiving the BATCH_END timestamp, outputs a vector of classification at the
|
||||
# BATCH_END timestamp.
|
||||
node {
|
||||
calculator: "EndLoopClassificationListCalculator"
|
||||
input_stream: "ITEM:single_handedness"
|
||||
input_stream: "BATCH_END:hand_rects_timestamp"
|
||||
output_stream: "ITERABLE:multi_handedness"
|
||||
}
|
||||
|
||||
# Calculate region of interest (ROI) based on detected hand landmarks to reuse
|
||||
# on the subsequent runs of the graph.
|
||||
node {
|
||||
calculator: "HandLandmarkLandmarksToRoi"
|
||||
input_stream: "IMAGE_SIZE:image_size_for_landmarks"
|
||||
input_stream: "LANDMARKS:single_hand_landmarks"
|
||||
output_stream: "ROI:single_hand_rect_from_landmarks"
|
||||
}
|
||||
|
||||
# Collects a set of landmarks for each hand into a vector. Upon receiving the
|
||||
# BATCH_END timestamp, outputs the vector of landmarks at the BATCH_END
|
||||
# timestamp.
|
||||
node {
|
||||
calculator: "EndLoopNormalizedLandmarkListVectorCalculator"
|
||||
input_stream: "ITEM:single_hand_landmarks"
|
||||
input_stream: "BATCH_END:hand_rects_timestamp"
|
||||
output_stream: "ITERABLE:multi_hand_landmarks"
|
||||
}
|
||||
|
||||
# Collects a NormalizedRect for each hand into a vector. Upon receiving the
|
||||
# BATCH_END timestamp, outputs the vector of NormalizedRect at the BATCH_END
|
||||
# timestamp.
|
||||
node {
|
||||
calculator: "EndLoopNormalizedRectCalculator"
|
||||
input_stream: "ITEM:single_hand_rect_from_landmarks"
|
||||
input_stream: "BATCH_END:hand_rects_timestamp"
|
||||
output_stream: "ITERABLE:hand_rects_from_landmarks"
|
||||
}
|
||||
|
||||
# Caches hand rects calculated from landmarks, and upon the arrival of the next
|
||||
# input image, sends out the cached rects with timestamps replaced by that of
|
||||
# the input image, essentially generating a packet that carries the previous
|
||||
# hand rects. Note that upon the arrival of the very first input image, a
|
||||
# timestamp bound update occurs to jump start the feedback loop.
|
||||
node {
|
||||
calculator: "PreviousLoopbackCalculator"
|
||||
input_stream: "MAIN:image"
|
||||
input_stream: "LOOP:hand_rects_from_landmarks"
|
||||
input_stream_info: {
|
||||
tag_index: "LOOP"
|
||||
back_edge: true
|
||||
}
|
||||
output_stream: "PREV_LOOP:prev_hand_rects_from_landmarks"
|
||||
}
|
||||
@@ -0,0 +1,241 @@
|
||||
# MediaPipe graph to detect/predict hand landmarks on GPU.
|
||||
#
|
||||
# The procedure is done in two steps:
|
||||
# - locate palms/hands
|
||||
# - detect landmarks for each palm/hand.
|
||||
# This graph tries to skip palm detection as much as possible by reusing
|
||||
# previously detected/predicted landmarks for new images.
|
||||
|
||||
type: "HandLandmarkTrackingGpu"
|
||||
|
||||
# GPU image. (GpuBuffer)
|
||||
input_stream: "IMAGE:image"
|
||||
|
||||
# Max number of hands to detect/track. (int)
|
||||
input_side_packet: "NUM_HANDS:num_hands"
|
||||
|
||||
# Whether palm detection can be skipped when hand regions can already be
|
||||
# approximated from hand landmarks on the previous frame.
|
||||
input_side_packet: "CAN_SKIP_DETECTION:can_skip_detection"
|
||||
|
||||
# Collection of detected/predicted hands, each represented as a list of
|
||||
# landmarks. (std::vector<NormalizedLandmarkList>)
|
||||
# NOTE: there will not be an output packet in the LANDMARKS stream for this
|
||||
# particular timestamp if none of hands detected. However, the MediaPipe
|
||||
# framework will internally inform the downstream calculators of the absence of
|
||||
# this packet so that they don't wait for it unnecessarily.
|
||||
output_stream: "LANDMARKS:multi_hand_landmarks"
|
||||
# Collection of handedness of the detected hands (i.e. is hand left or right),
|
||||
# each represented as a Classification proto.
|
||||
# Note that handedness is determined assuming the input image is mirrored,
|
||||
# i.e., taken with a front-facing/selfie camera with images flipped
|
||||
# horizontally.
|
||||
output_stream: "HANDEDNESS:multi_handedness"
|
||||
|
||||
# Extra outputs (for debugging, for instance).
|
||||
# Detected palms. (std::vector<Detection>)
|
||||
output_stream: "PALM_DETECTIONS:palm_detections"
|
||||
# Regions of interest calculated based on landmarks.
|
||||
# (std::vector<NormalizedRect>)
|
||||
output_stream: "HAND_ROIS_FROM_LANDMARKS:hand_rects"
|
||||
# Regions of interest calculated based on palm detections.
|
||||
# (std::vector<NormalizedRect>)
|
||||
output_stream: "HAND_ROIS_FROM_PALM_DETECTIONS:hand_rects_from_palm_detections"
|
||||
|
||||
# Determines if an input vector of NormalizedRect has a size greater than or
|
||||
# equal to the provided num_hands.
|
||||
node {
|
||||
calculator: "NormalizedRectVectorHasMinSizeCalculator"
|
||||
input_stream: "ITERABLE:prev_hand_rects_from_landmarks"
|
||||
input_side_packet: "num_hands"
|
||||
output_stream: "prev_has_enough_hands"
|
||||
}
|
||||
|
||||
# Drops the incoming image if the previous image had at least N hands.
|
||||
# and skipping palm detection is enabled.
|
||||
# Otherwise, passes the incoming image through to trigger a new round of palm
|
||||
# detection.
|
||||
node {
|
||||
calculator: "LogicCalculator"
|
||||
options: {
|
||||
[mediapipe.LogicCalculatorOptions.ext] { op: AND }
|
||||
}
|
||||
input_side_packet: "can_skip_detection"
|
||||
input_stream: "prev_has_enough_hands"
|
||||
output_stream: "skip_detection"
|
||||
}
|
||||
node {
|
||||
calculator: "GateCalculator"
|
||||
input_stream: "image"
|
||||
input_stream: "DISALLOW:skip_detection"
|
||||
output_stream: "palm_detection_image"
|
||||
options: {
|
||||
[mediapipe.GateCalculatorOptions.ext] {
|
||||
empty_packets_as_allow: true
|
||||
}
|
||||
}
|
||||
}
|
||||
node {
|
||||
calculator: "GateCalculator"
|
||||
input_stream: "prev_hand_rects_from_landmarks"
|
||||
input_stream: "ALLOW:skip_detection"
|
||||
output_stream: "gated_prev_hand_rects_from_landmarks"
|
||||
}
|
||||
|
||||
# Detects palms.
|
||||
node {
|
||||
calculator: "PalmDetectionGpu"
|
||||
input_stream: "IMAGE:palm_detection_image"
|
||||
output_stream: "DETECTIONS:all_palm_detections"
|
||||
}
|
||||
|
||||
# Makes sure there are no more detections than provided num_hands.
|
||||
node {
|
||||
calculator: "ClipDetectionVectorSizeCalculator"
|
||||
input_stream: "all_palm_detections"
|
||||
output_stream: "palm_detections"
|
||||
input_side_packet: "num_hands"
|
||||
}
|
||||
|
||||
# Extracts image size.
|
||||
node {
|
||||
calculator: "ImagePropertiesCalculator"
|
||||
input_stream: "IMAGE_GPU:palm_detection_image"
|
||||
output_stream: "SIZE:palm_detection_image_size"
|
||||
}
|
||||
|
||||
# Outputs each element of palm_detections at a fake timestamp for the rest of
|
||||
# the graph to process. Clones the image_size packet for each palm_detection at
|
||||
# the fake timestamp. At the end of the loop, outputs the BATCH_END timestamp
|
||||
# for downstream calculators to inform them that all elements in the vector have
|
||||
# been processed.
|
||||
node {
|
||||
calculator: "BeginLoopDetectionCalculator"
|
||||
input_stream: "ITERABLE:palm_detections"
|
||||
input_stream: "CLONE:palm_detection_image_size"
|
||||
output_stream: "ITEM:palm_detection"
|
||||
output_stream: "CLONE:image_size_for_palms"
|
||||
output_stream: "BATCH_END:palm_detections_timestamp"
|
||||
}
|
||||
|
||||
# Calculates region of interest (ROI) base on the specified palm.
|
||||
node {
|
||||
calculator: "PalmDetectionDetectionToRoi"
|
||||
input_stream: "DETECTION:palm_detection"
|
||||
input_stream: "IMAGE_SIZE:image_size_for_palms"
|
||||
output_stream: "ROI:hand_rect_from_palm_detection"
|
||||
}
|
||||
|
||||
# Collects a NormalizedRect for each hand into a vector. Upon receiving the
|
||||
# BATCH_END timestamp, outputs the vector of NormalizedRect at the BATCH_END
|
||||
# timestamp.
|
||||
node {
|
||||
name: "EndLoopForPalmDetections"
|
||||
calculator: "EndLoopNormalizedRectCalculator"
|
||||
input_stream: "ITEM:hand_rect_from_palm_detection"
|
||||
input_stream: "BATCH_END:palm_detections_timestamp"
|
||||
output_stream: "ITERABLE:hand_rects_from_palm_detections"
|
||||
}
|
||||
|
||||
# Performs association between NormalizedRect vector elements from previous
|
||||
# image and rects based on palm detections from the current image. This
|
||||
# calculator ensures that the output hand_rects vector doesn't contain
|
||||
# overlapping regions based on the specified min_similarity_threshold.
|
||||
node {
|
||||
calculator: "AssociationNormRectCalculator"
|
||||
input_stream: "gated_prev_hand_rects_from_landmarks"
|
||||
input_stream: "hand_rects_from_palm_detections"
|
||||
output_stream: "hand_rects"
|
||||
options: {
|
||||
[mediapipe.AssociationCalculatorOptions.ext] {
|
||||
min_similarity_threshold: 0.5
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Extracts image size.
|
||||
node {
|
||||
calculator: "ImagePropertiesCalculator"
|
||||
input_stream: "IMAGE_GPU:image"
|
||||
output_stream: "SIZE:image_size"
|
||||
}
|
||||
|
||||
# Outputs each element of hand_rects at a fake timestamp for the rest of the
|
||||
# graph to process. Clones image and image size packets for each
|
||||
# single_hand_rect at the fake timestamp. At the end of the loop, outputs the
|
||||
# BATCH_END timestamp for downstream calculators to inform them that all
|
||||
# elements in the vector have been processed.
|
||||
node {
|
||||
calculator: "BeginLoopNormalizedRectCalculator"
|
||||
input_stream: "ITERABLE:hand_rects"
|
||||
input_stream: "CLONE:0:image"
|
||||
input_stream: "CLONE:1:image_size"
|
||||
output_stream: "ITEM:single_hand_rect"
|
||||
output_stream: "CLONE:0:image_for_landmarks"
|
||||
output_stream: "CLONE:1:image_size_for_landmarks"
|
||||
output_stream: "BATCH_END:hand_rects_timestamp"
|
||||
}
|
||||
|
||||
# Detect hand landmarks for the specific hand rect.
|
||||
node {
|
||||
calculator: "HandLandmarkGpu"
|
||||
input_stream: "IMAGE:image_for_landmarks"
|
||||
input_stream: "ROI:single_hand_rect"
|
||||
output_stream: "LANDMARKS:single_hand_landmarks"
|
||||
output_stream: "HANDEDNESS:single_handedness"
|
||||
}
|
||||
|
||||
# Collects the handedness for each single hand into a vector. Upon
|
||||
# receiving the BATCH_END timestamp, outputs a vector of classification at the
|
||||
# BATCH_END timestamp.
|
||||
node {
|
||||
calculator: "EndLoopClassificationListCalculator"
|
||||
input_stream: "ITEM:single_handedness"
|
||||
input_stream: "BATCH_END:hand_rects_timestamp"
|
||||
output_stream: "ITERABLE:multi_handedness"
|
||||
}
|
||||
|
||||
# Calculate region of interest (ROI) based on detected hand landmarks to reuse
|
||||
# on the subsequent runs of the graph.
|
||||
node {
|
||||
calculator: "HandLandmarkLandmarksToRoi"
|
||||
input_stream: "IMAGE_SIZE:image_size_for_landmarks"
|
||||
input_stream: "LANDMARKS:single_hand_landmarks"
|
||||
output_stream: "ROI:single_hand_rect_from_landmarks"
|
||||
}
|
||||
|
||||
# Collects a set of landmarks for each hand into a vector. Upon receiving the
|
||||
# BATCH_END timestamp, outputs the vector of landmarks at the BATCH_END
|
||||
# timestamp.
|
||||
node {
|
||||
calculator: "EndLoopNormalizedLandmarkListVectorCalculator"
|
||||
input_stream: "ITEM:single_hand_landmarks"
|
||||
input_stream: "BATCH_END:hand_rects_timestamp"
|
||||
output_stream: "ITERABLE:multi_hand_landmarks"
|
||||
}
|
||||
|
||||
# Collects a NormalizedRect for each hand into a vector. Upon receiving the
|
||||
# BATCH_END timestamp, outputs the vector of NormalizedRect at the BATCH_END
|
||||
# timestamp.
|
||||
node {
|
||||
calculator: "EndLoopNormalizedRectCalculator"
|
||||
input_stream: "ITEM:single_hand_rect_from_landmarks"
|
||||
input_stream: "BATCH_END:hand_rects_timestamp"
|
||||
output_stream: "ITERABLE:hand_rects_from_landmarks"
|
||||
}
|
||||
|
||||
# Caches hand rects calculated from landmarks, and upon the arrival of the next
|
||||
# input image, sends out the cached rects with timestamps replaced by that of
|
||||
# the input image, essentially generating a packet that carries the previous
|
||||
# hand rects. Note that upon the arrival of the very first input image, a
|
||||
# timestamp bound update occurs to jump start the feedback loop.
|
||||
node {
|
||||
calculator: "PreviousLoopbackCalculator"
|
||||
input_stream: "MAIN:image"
|
||||
input_stream: "LOOP:hand_rects_from_landmarks"
|
||||
input_stream_info: {
|
||||
tag_index: "LOOP"
|
||||
back_edge: true
|
||||
}
|
||||
output_stream: "PREV_LOOP:prev_hand_rects_from_landmarks"
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
Left
|
||||
Right
|
||||
@@ -0,0 +1,47 @@
|
||||
# MediaPipe subgraph that calculates hand ROI from palm detection.
|
||||
|
||||
type: "PalmDetectionDetectionToRoi"
|
||||
|
||||
# Palm detection. (Detection)
|
||||
input_stream: "DETECTION:detection"
|
||||
# Frame size. (std::pair<int, int>)
|
||||
input_stream: "IMAGE_SIZE:image_size"
|
||||
|
||||
# ROI (region of interest) according to landmarks, represented as normalized
|
||||
# rect. (NormalizedRect)
|
||||
output_stream: "ROI:roi"
|
||||
|
||||
# Converts results of palm detection into a rectangle (normalized by image size)
|
||||
# that encloses the palm and is rotated such that the line connecting center of
|
||||
# the wrist and MCP of the middle finger is aligned with the Y-axis of the
|
||||
# rectangle.
|
||||
node {
|
||||
calculator: "DetectionsToRectsCalculator"
|
||||
input_stream: "DETECTION:detection"
|
||||
input_stream: "IMAGE_SIZE:image_size"
|
||||
output_stream: "NORM_RECT:raw_roi"
|
||||
options: {
|
||||
[mediapipe.DetectionsToRectsCalculatorOptions.ext] {
|
||||
rotation_vector_start_keypoint_index: 0 # Center of wrist.
|
||||
rotation_vector_end_keypoint_index: 2 # MCP of middle finger.
|
||||
rotation_vector_target_angle_degrees: 90
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Expands and shifts the rectangle that contains the palm so that it's likely
|
||||
# to cover the entire hand.
|
||||
node {
|
||||
calculator: "RectTransformationCalculator"
|
||||
input_stream: "NORM_RECT:raw_roi"
|
||||
input_stream: "IMAGE_SIZE:image_size"
|
||||
output_stream: "roi"
|
||||
options: {
|
||||
[mediapipe.RectTransformationCalculatorOptions.ext] {
|
||||
scale_x: 2.6
|
||||
scale_y: 2.6
|
||||
shift_y: -0.5
|
||||
square_long: true
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
# Copyright 2020 The MediaPipe Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
load(
|
||||
"//mediapipe/framework/tool:mediapipe_graph.bzl",
|
||||
"mediapipe_simple_subgraph",
|
||||
)
|
||||
|
||||
licenses(["notice"])
|
||||
|
||||
package(default_visibility = ["//visibility:public"])
|
||||
|
||||
exports_files(["palm_detection.tflite"])
|
||||
|
||||
mediapipe_simple_subgraph(
|
||||
name = "palm_detection_cpu",
|
||||
graph = "palm_detection_cpu.pbtxt",
|
||||
register_as = "PalmDetectionCpu",
|
||||
deps = [
|
||||
"//mediapipe/calculators/tensor:image_to_tensor_calculator",
|
||||
"//mediapipe/calculators/tensor:inference_calculator",
|
||||
"//mediapipe/calculators/tensor:tensors_to_detections_calculator",
|
||||
"//mediapipe/calculators/tflite:ssd_anchors_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_custom_op_resolver_calculator",
|
||||
"//mediapipe/calculators/util:detection_letterbox_removal_calculator",
|
||||
"//mediapipe/calculators/util:non_max_suppression_calculator",
|
||||
],
|
||||
)
|
||||
|
||||
mediapipe_simple_subgraph(
|
||||
name = "palm_detection_gpu",
|
||||
graph = "palm_detection_gpu.pbtxt",
|
||||
register_as = "PalmDetectionGpu",
|
||||
deps = [
|
||||
"//mediapipe/calculators/tensor:image_to_tensor_calculator",
|
||||
"//mediapipe/calculators/tensor:inference_calculator",
|
||||
"//mediapipe/calculators/tensor:tensors_to_detections_calculator",
|
||||
"//mediapipe/calculators/tflite:ssd_anchors_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_custom_op_resolver_calculator",
|
||||
"//mediapipe/calculators/util:detection_letterbox_removal_calculator",
|
||||
"//mediapipe/calculators/util:non_max_suppression_calculator",
|
||||
],
|
||||
)
|
||||
|
||||
exports_files(
|
||||
srcs = [
|
||||
"palm_detection.tflite",
|
||||
"palm_detection_labelmap.txt",
|
||||
],
|
||||
)
|
||||
@@ -0,0 +1,7 @@
|
||||
# palm_detection
|
||||
|
||||
Subgraphs|Details
|
||||
:--- | :---
|
||||
[`PalmDetectionCpu`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/palm_detection/palm_detection_cpu.pbtxt)| Detects palms/hands. (CPU input.)
|
||||
[`PalmDetectionGpu`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/palm_detection/palm_detection_gpu.pbtxt)| Detects palms/hands. (GPU input.)
|
||||
|
||||
BIN
Binary file not shown.
@@ -0,0 +1,134 @@
|
||||
# MediaPipe graph to detect palms with TensorFlow Lite on CPU.
|
||||
|
||||
type: "PalmDetectionCpu"
|
||||
|
||||
# CPU image. (ImageFrame)
|
||||
input_stream: "IMAGE:image"
|
||||
|
||||
# Detected palms. (std::vector<Detection>)
|
||||
# NOTE: there will not be an output packet in the DETECTIONS stream for this
|
||||
# particular timestamp if none of palms detected. However, the MediaPipe
|
||||
# framework will internally inform the downstream calculators of the absence of
|
||||
# this packet so that they don't wait for it unnecessarily.
|
||||
output_stream: "DETECTIONS:detections"
|
||||
|
||||
# Transforms an image into a 128x128 tensor while keeping the aspect ratio, and
|
||||
# therefore may result in potential letterboxing.
|
||||
node {
|
||||
calculator: "ImageToTensorCalculator"
|
||||
input_stream: "IMAGE:image"
|
||||
output_stream: "TENSORS:input_tensor"
|
||||
output_stream: "LETTERBOX_PADDING:letterbox_padding"
|
||||
options: {
|
||||
[mediapipe.ImageToTensorCalculatorOptions.ext] {
|
||||
output_tensor_width: 128
|
||||
output_tensor_height: 128
|
||||
keep_aspect_ratio: true
|
||||
output_tensor_float_range {
|
||||
min: -1.0
|
||||
max: 1.0
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
# Generates a single side packet containing a TensorFlow Lite op resolver that
|
||||
# supports custom ops needed by the model used in this graph.
|
||||
node {
|
||||
calculator: "TfLiteCustomOpResolverCalculator"
|
||||
output_side_packet: "opresolver"
|
||||
}
|
||||
|
||||
# Runs a TensorFlow Lite model on CPU that takes an image tensor and outputs a
|
||||
# vector of tensors representing, for instance, detection boxes/keypoints and
|
||||
# scores.
|
||||
node {
|
||||
calculator: "InferenceCalculator"
|
||||
input_stream: "TENSORS:input_tensor"
|
||||
output_stream: "TENSORS:detection_tensors"
|
||||
input_side_packet: "CUSTOM_OP_RESOLVER:opresolver"
|
||||
options: {
|
||||
[mediapipe.InferenceCalculatorOptions.ext] {
|
||||
model_path: "mediapipe/modules/palm_detection/palm_detection.tflite"
|
||||
delegate { xnnpack {} }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Generates a single side packet containing a vector of SSD anchors based on
|
||||
# the specification in the options.
|
||||
node {
|
||||
calculator: "SsdAnchorsCalculator"
|
||||
output_side_packet: "anchors"
|
||||
options: {
|
||||
[mediapipe.SsdAnchorsCalculatorOptions.ext] {
|
||||
num_layers: 4
|
||||
min_scale: 0.1484375
|
||||
max_scale: 0.75
|
||||
input_size_height: 128
|
||||
input_size_width: 128
|
||||
anchor_offset_x: 0.5
|
||||
anchor_offset_y: 0.5
|
||||
strides: 8
|
||||
strides: 16
|
||||
strides: 16
|
||||
strides: 16
|
||||
aspect_ratios: 1.0
|
||||
fixed_anchor_size: true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Decodes the detection tensors generated by the TensorFlow Lite model, based on
|
||||
# the SSD anchors and the specification in the options, into a vector of
|
||||
# detections. Each detection describes a detected object.
|
||||
node {
|
||||
calculator: "TensorsToDetectionsCalculator"
|
||||
input_stream: "TENSORS:detection_tensors"
|
||||
input_side_packet: "ANCHORS:anchors"
|
||||
output_stream: "DETECTIONS:unfiltered_detections"
|
||||
options: {
|
||||
[mediapipe.TensorsToDetectionsCalculatorOptions.ext] {
|
||||
num_classes: 1
|
||||
num_boxes: 896
|
||||
num_coords: 18
|
||||
box_coord_offset: 0
|
||||
keypoint_coord_offset: 4
|
||||
num_keypoints: 7
|
||||
num_values_per_keypoint: 2
|
||||
sigmoid_score: true
|
||||
score_clipping_thresh: 100.0
|
||||
reverse_output_order: true
|
||||
|
||||
x_scale: 128.0
|
||||
y_scale: 128.0
|
||||
h_scale: 128.0
|
||||
w_scale: 128.0
|
||||
min_score_thresh: 0.5
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Performs non-max suppression to remove excessive detections.
|
||||
node {
|
||||
calculator: "NonMaxSuppressionCalculator"
|
||||
input_stream: "unfiltered_detections"
|
||||
output_stream: "filtered_detections"
|
||||
options: {
|
||||
[mediapipe.NonMaxSuppressionCalculatorOptions.ext] {
|
||||
min_suppression_threshold: 0.3
|
||||
overlap_type: INTERSECTION_OVER_UNION
|
||||
algorithm: WEIGHTED
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Adjusts detection locations (already normalized to [0.f, 1.f]) on the
|
||||
# letterboxed image (after image transformation with the FIT scale mode) to the
|
||||
# corresponding locations on the same image with the letterbox removed (the
|
||||
# input image to the graph before image transformation).
|
||||
node {
|
||||
calculator: "DetectionLetterboxRemovalCalculator"
|
||||
input_stream: "DETECTIONS:filtered_detections"
|
||||
input_stream: "LETTERBOX_PADDING:letterbox_padding"
|
||||
output_stream: "DETECTIONS:detections"
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
# MediaPipe graph to detect palms with TensorFlow Lite on GPU.
|
||||
|
||||
type: "PalmDetectionGpu"
|
||||
|
||||
# GPU image. (GpuBuffer)
|
||||
input_stream: "IMAGE:image"
|
||||
|
||||
# Detected palms. (std::vector<Detection>)
|
||||
# NOTE: there will not be an output packet in the DETECTIONS stream for this
|
||||
# particular timestamp if none of palms detected. However, the MediaPipe
|
||||
# framework will internally inform the downstream calculators of the absence of
|
||||
# this packet so that they don't wait for it unnecessarily.
|
||||
output_stream: "DETECTIONS:detections"
|
||||
|
||||
# Transforms an image into a 256x256 tensor while keeping the aspect ratio, and
|
||||
# therefore may result in potential letterboxing.
|
||||
node {
|
||||
calculator: "ImageToTensorCalculator"
|
||||
input_stream: "IMAGE_GPU:image"
|
||||
output_stream: "TENSORS:input_tensor"
|
||||
output_stream: "LETTERBOX_PADDING:letterbox_padding"
|
||||
options: {
|
||||
[mediapipe.ImageToTensorCalculatorOptions.ext] {
|
||||
output_tensor_width: 128
|
||||
output_tensor_height: 128
|
||||
keep_aspect_ratio: true
|
||||
output_tensor_float_range {
|
||||
min: -1.0
|
||||
max: 1.0
|
||||
}
|
||||
gpu_origin: TOP_LEFT
|
||||
}
|
||||
}
|
||||
}
|
||||
# Generates a single side packet containing a TensorFlow Lite op resolver that
|
||||
# supports custom ops needed by the model used in this graph.
|
||||
node {
|
||||
calculator: "TfLiteCustomOpResolverCalculator"
|
||||
output_side_packet: "opresolver"
|
||||
options: {
|
||||
[mediapipe.TfLiteCustomOpResolverCalculatorOptions.ext] {
|
||||
use_gpu: true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Runs a TensorFlow Lite model on GPU that takes an image tensor and outputs a
|
||||
# vector of tensors representing, for instance, detection boxes/keypoints and
|
||||
# scores.
|
||||
node {
|
||||
calculator: "InferenceCalculator"
|
||||
input_stream: "TENSORS:input_tensor"
|
||||
output_stream: "TENSORS:detection_tensors"
|
||||
input_side_packet: "CUSTOM_OP_RESOLVER:opresolver"
|
||||
options: {
|
||||
[mediapipe.InferenceCalculatorOptions.ext] {
|
||||
model_path: "mediapipe/modules/palm_detection/palm_detection.tflite"
|
||||
use_gpu: true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Generates a single side packet containing a vector of SSD anchors based on
|
||||
# the specification in the options.
|
||||
node {
|
||||
calculator: "SsdAnchorsCalculator"
|
||||
output_side_packet: "anchors"
|
||||
options: {
|
||||
[mediapipe.SsdAnchorsCalculatorOptions.ext] {
|
||||
num_layers: 4
|
||||
min_scale: 0.1484375
|
||||
max_scale: 0.75
|
||||
input_size_height: 128
|
||||
input_size_width: 128
|
||||
anchor_offset_x: 0.5
|
||||
anchor_offset_y: 0.5
|
||||
strides: 8
|
||||
strides: 16
|
||||
strides: 16
|
||||
strides: 16
|
||||
aspect_ratios: 1.0
|
||||
fixed_anchor_size: true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Decodes the detection tensors generated by the TensorFlow Lite model, based on
|
||||
# the SSD anchors and the specification in the options, into a vector of
|
||||
# detections. Each detection describes a detected object.
|
||||
node {
|
||||
calculator: "TensorsToDetectionsCalculator"
|
||||
input_stream: "TENSORS:detection_tensors"
|
||||
input_side_packet: "ANCHORS:anchors"
|
||||
output_stream: "DETECTIONS:unfiltered_detections"
|
||||
options: {
|
||||
[mediapipe.TensorsToDetectionsCalculatorOptions.ext] {
|
||||
num_classes: 1
|
||||
num_boxes: 896
|
||||
num_coords: 18
|
||||
box_coord_offset: 0
|
||||
keypoint_coord_offset: 4
|
||||
num_keypoints: 7
|
||||
num_values_per_keypoint: 2
|
||||
sigmoid_score: true
|
||||
score_clipping_thresh: 100.0
|
||||
reverse_output_order: true
|
||||
|
||||
x_scale: 128.0
|
||||
y_scale: 128.0
|
||||
h_scale: 128.0
|
||||
w_scale: 128.0
|
||||
min_score_thresh: 0.5
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Performs non-max suppression to remove excessive detections.
|
||||
node {
|
||||
calculator: "NonMaxSuppressionCalculator"
|
||||
input_stream: "unfiltered_detections"
|
||||
output_stream: "filtered_detections"
|
||||
options: {
|
||||
[mediapipe.NonMaxSuppressionCalculatorOptions.ext] {
|
||||
min_suppression_threshold: 0.3
|
||||
overlap_type: INTERSECTION_OVER_UNION
|
||||
algorithm: WEIGHTED
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Adjusts detection locations (already normalized to [0.f, 1.f]) on the
|
||||
# letterboxed image (after image transformation with the FIT scale mode) to the
|
||||
# corresponding locations on the same image with the letterbox removed (the
|
||||
# input image to the graph before image transformation).
|
||||
node {
|
||||
calculator: "DetectionLetterboxRemovalCalculator"
|
||||
input_stream: "DETECTIONS:filtered_detections"
|
||||
input_stream: "LETTERBOX_PADDING:letterbox_padding"
|
||||
output_stream: "DETECTIONS:detections"
|
||||
}
|
||||
@@ -26,11 +26,10 @@ mediapipe_simple_subgraph(
|
||||
graph = "pose_detection_cpu.pbtxt",
|
||||
register_as = "PoseDetectionCpu",
|
||||
deps = [
|
||||
"//mediapipe/calculators/image:image_transformation_calculator",
|
||||
"//mediapipe/calculators/tensor:image_to_tensor_calculator",
|
||||
"//mediapipe/calculators/tensor:inference_calculator",
|
||||
"//mediapipe/calculators/tensor:tensors_to_detections_calculator",
|
||||
"//mediapipe/calculators/tflite:ssd_anchors_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_converter_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_inference_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_tensors_to_detections_calculator",
|
||||
"//mediapipe/calculators/util:detection_letterbox_removal_calculator",
|
||||
"//mediapipe/calculators/util:non_max_suppression_calculator",
|
||||
],
|
||||
@@ -41,11 +40,10 @@ mediapipe_simple_subgraph(
|
||||
graph = "pose_detection_gpu.pbtxt",
|
||||
register_as = "PoseDetectionGpu",
|
||||
deps = [
|
||||
"//mediapipe/calculators/image:image_transformation_calculator",
|
||||
"//mediapipe/calculators/tensor:image_to_tensor_calculator",
|
||||
"//mediapipe/calculators/tensor:inference_calculator",
|
||||
"//mediapipe/calculators/tensor:tensors_to_detections_calculator",
|
||||
"//mediapipe/calculators/tflite:ssd_anchors_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_converter_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_inference_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_tensors_to_detections_calculator",
|
||||
"//mediapipe/calculators/util:detection_letterbox_removal_calculator",
|
||||
"//mediapipe/calculators/util:non_max_suppression_calculator",
|
||||
],
|
||||
|
||||
@@ -36,40 +36,36 @@ input_stream: "IMAGE:image"
|
||||
# this packet so that they don't wait for it unnecessarily.
|
||||
output_stream: "DETECTIONS:detections"
|
||||
|
||||
# Transforms the input image on CPU to a 128x128 image. To scale the input
|
||||
# image, the scale_mode option is set to FIT to preserve the aspect ratio,
|
||||
# resulting in potential letterboxing in the transformed image.
|
||||
# Transforms the input image into a 128x128 while keeping the aspect ratio
|
||||
# (what is expected by the corresponding model), resulting in potential
|
||||
# letterboxing in the transformed image.
|
||||
node: {
|
||||
calculator: "ImageTransformationCalculator"
|
||||
calculator: "ImageToTensorCalculator"
|
||||
input_stream: "IMAGE:image"
|
||||
output_stream: "IMAGE:transformed_image"
|
||||
output_stream: "TENSORS:input_tensors"
|
||||
output_stream: "LETTERBOX_PADDING:letterbox_padding"
|
||||
options: {
|
||||
[mediapipe.ImageTransformationCalculatorOptions.ext] {
|
||||
output_width: 128
|
||||
output_height: 128
|
||||
scale_mode: FIT
|
||||
[mediapipe.ImageToTensorCalculatorOptions.ext] {
|
||||
output_tensor_width: 128
|
||||
output_tensor_height: 128
|
||||
keep_aspect_ratio: true
|
||||
output_tensor_float_range {
|
||||
min: -1.0
|
||||
max: 1.0
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Converts the transformed input image on CPU into an image tensor stored as a
|
||||
# TfLiteTensor.
|
||||
node {
|
||||
calculator: "TfLiteConverterCalculator"
|
||||
input_stream: "IMAGE:transformed_image"
|
||||
output_stream: "TENSORS:input_tensors"
|
||||
}
|
||||
|
||||
# Runs a TensorFlow Lite model on CPU that takes an image tensor and outputs a
|
||||
# vector of tensors representing, for instance, detection boxes/keypoints and
|
||||
# scores.
|
||||
node {
|
||||
calculator: "TfLiteInferenceCalculator"
|
||||
calculator: "InferenceCalculator"
|
||||
input_stream: "TENSORS:input_tensors"
|
||||
output_stream: "TENSORS:detection_tensors"
|
||||
options: {
|
||||
[mediapipe.TfLiteInferenceCalculatorOptions.ext] {
|
||||
[mediapipe.InferenceCalculatorOptions.ext] {
|
||||
model_path: "mediapipe/modules/pose_detection/pose_detection.tflite"
|
||||
delegate { xnnpack {} }
|
||||
}
|
||||
@@ -104,12 +100,12 @@ node {
|
||||
# the SSD anchors and the specification in the options, into a vector of
|
||||
# detections. Each detection describes a detected object.
|
||||
node {
|
||||
calculator: "TfLiteTensorsToDetectionsCalculator"
|
||||
calculator: "TensorsToDetectionsCalculator"
|
||||
input_stream: "TENSORS:detection_tensors"
|
||||
input_side_packet: "ANCHORS:anchors"
|
||||
output_stream: "DETECTIONS:unfiltered_detections"
|
||||
options: {
|
||||
[mediapipe.TfLiteTensorsToDetectionsCalculatorOptions.ext] {
|
||||
[mediapipe.TensorsToDetectionsCalculatorOptions.ext] {
|
||||
num_classes: 1
|
||||
num_boxes: 896
|
||||
num_coords: 12
|
||||
|
||||
@@ -36,41 +36,37 @@ input_stream: "IMAGE:image"
|
||||
# this packet so that they don't wait for it unnecessarily.
|
||||
output_stream: "DETECTIONS:detections"
|
||||
|
||||
# Transforms the input image on GPU to a 128x128 image. To scale the input
|
||||
# image, the scale_mode option is set to FIT to preserve the aspect ratio,
|
||||
# resulting in potential letterboxing in the transformed image.
|
||||
# Transforms the input image into a 128x128 while keeping the aspect ratio
|
||||
# (what is expected by the corresponding model), resulting in potential
|
||||
# letterboxing in the transformed image.
|
||||
node: {
|
||||
calculator: "ImageTransformationCalculator"
|
||||
calculator: "ImageToTensorCalculator"
|
||||
input_stream: "IMAGE_GPU:image"
|
||||
output_stream: "IMAGE_GPU:transformed_image"
|
||||
output_stream: "TENSORS:input_tensors"
|
||||
output_stream: "LETTERBOX_PADDING:letterbox_padding"
|
||||
options: {
|
||||
[mediapipe.ImageTransformationCalculatorOptions.ext] {
|
||||
output_width: 128
|
||||
output_height: 128
|
||||
scale_mode: FIT
|
||||
[mediapipe.ImageToTensorCalculatorOptions.ext] {
|
||||
output_tensor_width: 128
|
||||
output_tensor_height: 128
|
||||
keep_aspect_ratio: true
|
||||
output_tensor_float_range {
|
||||
min: -1.0
|
||||
max: 1.0
|
||||
}
|
||||
gpu_origin: TOP_LEFT
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Converts the transformed input image on GPU into an image tensor stored as a
|
||||
# TfLiteTensor.
|
||||
node {
|
||||
calculator: "TfLiteConverterCalculator"
|
||||
input_stream: "IMAGE_GPU:transformed_image"
|
||||
output_stream: "TENSORS_GPU:input_tensors"
|
||||
}
|
||||
|
||||
# Runs a TensorFlow Lite model on GPU that takes an image tensor and outputs a
|
||||
# Runs a TensorFlow Lite model on CPU that takes an image tensor and outputs a
|
||||
# vector of tensors representing, for instance, detection boxes/keypoints and
|
||||
# scores.
|
||||
node {
|
||||
calculator: "TfLiteInferenceCalculator"
|
||||
input_stream: "TENSORS_GPU:input_tensors"
|
||||
# TODO: we can use TENSORS_GPU here and in the downstream calculator
|
||||
calculator: "InferenceCalculator"
|
||||
input_stream: "TENSORS:input_tensors"
|
||||
output_stream: "TENSORS:detection_tensors"
|
||||
options: {
|
||||
[mediapipe.TfLiteInferenceCalculatorOptions.ext] {
|
||||
[mediapipe.InferenceCalculatorOptions.ext] {
|
||||
model_path: "mediapipe/modules/pose_detection/pose_detection.tflite"
|
||||
}
|
||||
}
|
||||
@@ -104,12 +100,12 @@ node {
|
||||
# the SSD anchors and the specification in the options, into a vector of
|
||||
# detections. Each detection describes a detected object.
|
||||
node {
|
||||
calculator: "TfLiteTensorsToDetectionsCalculator"
|
||||
calculator: "TensorsToDetectionsCalculator"
|
||||
input_stream: "TENSORS:detection_tensors"
|
||||
input_side_packet: "ANCHORS:anchors"
|
||||
output_stream: "DETECTIONS:unfiltered_detections"
|
||||
options: {
|
||||
[mediapipe.TfLiteTensorsToDetectionsCalculatorOptions.ext] {
|
||||
[mediapipe.TensorsToDetectionsCalculatorOptions.ext] {
|
||||
num_classes: 1
|
||||
num_boxes: 896
|
||||
num_coords: 12
|
||||
|
||||
@@ -27,13 +27,12 @@ mediapipe_simple_subgraph(
|
||||
register_as = "PoseLandmarkUpperBodyByRoiGpu",
|
||||
deps = [
|
||||
"//mediapipe/calculators/core:gate_calculator",
|
||||
"//mediapipe/calculators/core:split_normalized_landmark_list_calculator",
|
||||
"//mediapipe/calculators/core:split_vector_calculator",
|
||||
"//mediapipe/calculators/image:image_cropping_calculator",
|
||||
"//mediapipe/calculators/image:image_transformation_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_converter_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_inference_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_tensors_to_floats_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_tensors_to_landmarks_calculator",
|
||||
"//mediapipe/calculators/tensor:image_to_tensor_calculator",
|
||||
"//mediapipe/calculators/tensor:inference_calculator",
|
||||
"//mediapipe/calculators/tensor:tensors_to_floats_calculator",
|
||||
"//mediapipe/calculators/tensor:tensors_to_landmarks_calculator",
|
||||
"//mediapipe/calculators/util:landmark_letterbox_removal_calculator",
|
||||
"//mediapipe/calculators/util:landmark_projection_calculator",
|
||||
"//mediapipe/calculators/util:thresholding_calculator",
|
||||
@@ -46,13 +45,12 @@ mediapipe_simple_subgraph(
|
||||
register_as = "PoseLandmarkUpperBodyByRoiCpu",
|
||||
deps = [
|
||||
"//mediapipe/calculators/core:gate_calculator",
|
||||
"//mediapipe/calculators/core:split_normalized_landmark_list_calculator",
|
||||
"//mediapipe/calculators/core:split_vector_calculator",
|
||||
"//mediapipe/calculators/image:image_cropping_calculator",
|
||||
"//mediapipe/calculators/image:image_transformation_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_converter_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_inference_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_tensors_to_floats_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_tensors_to_landmarks_calculator",
|
||||
"//mediapipe/calculators/tensor:image_to_tensor_calculator",
|
||||
"//mediapipe/calculators/tensor:inference_calculator",
|
||||
"//mediapipe/calculators/tensor:tensors_to_floats_calculator",
|
||||
"//mediapipe/calculators/tensor:tensors_to_landmarks_calculator",
|
||||
"//mediapipe/calculators/util:landmark_letterbox_removal_calculator",
|
||||
"//mediapipe/calculators/util:landmark_projection_calculator",
|
||||
"//mediapipe/calculators/util:thresholding_calculator",
|
||||
@@ -73,6 +71,7 @@ mediapipe_simple_subgraph(
|
||||
"//mediapipe/calculators/core:previous_loopback_calculator",
|
||||
"//mediapipe/calculators/core:split_vector_calculator",
|
||||
"//mediapipe/calculators/image:image_properties_calculator",
|
||||
"//mediapipe/calculators/util:logic_calculator",
|
||||
"//mediapipe/modules/pose_detection:pose_detection_gpu",
|
||||
],
|
||||
)
|
||||
@@ -91,10 +90,33 @@ mediapipe_simple_subgraph(
|
||||
"//mediapipe/calculators/core:previous_loopback_calculator",
|
||||
"//mediapipe/calculators/core:split_vector_calculator",
|
||||
"//mediapipe/calculators/image:image_properties_calculator",
|
||||
"//mediapipe/calculators/util:logic_calculator",
|
||||
"//mediapipe/modules/pose_detection:pose_detection_cpu",
|
||||
],
|
||||
)
|
||||
|
||||
mediapipe_simple_subgraph(
|
||||
name = "pose_landmark_upper_body_smoothed_cpu",
|
||||
graph = "pose_landmark_upper_body_smoothed_cpu.pbtxt",
|
||||
register_as = "PoseLandmarkUpperBodySmoothedCpu",
|
||||
deps = [
|
||||
":pose_landmark_upper_body_cpu",
|
||||
"//mediapipe/calculators/image:image_properties_calculator",
|
||||
"//mediapipe/calculators/util:landmarks_smoothing_calculator",
|
||||
],
|
||||
)
|
||||
|
||||
mediapipe_simple_subgraph(
|
||||
name = "pose_landmark_upper_body_smoothed_gpu",
|
||||
graph = "pose_landmark_upper_body_smoothed_gpu.pbtxt",
|
||||
register_as = "PoseLandmarkUpperBodySmoothedGpu",
|
||||
deps = [
|
||||
":pose_landmark_upper_body_gpu",
|
||||
"//mediapipe/calculators/image:image_properties_calculator",
|
||||
"//mediapipe/calculators/util:landmarks_smoothing_calculator",
|
||||
],
|
||||
)
|
||||
|
||||
exports_files(
|
||||
srcs = [
|
||||
"pose_landmark_upper_body.tflite",
|
||||
@@ -116,7 +138,6 @@ mediapipe_simple_subgraph(
|
||||
graph = "pose_landmark_upper_body_landmarks_to_roi.pbtxt",
|
||||
register_as = "PoseLandmarkUpperBodyLandmarksToRoi",
|
||||
deps = [
|
||||
"//mediapipe/calculators/core:split_vector_calculator",
|
||||
"//mediapipe/calculators/util:alignment_points_to_rects_calculator",
|
||||
"//mediapipe/calculators/util:landmarks_to_detection_calculator",
|
||||
"//mediapipe/calculators/util:rect_transformation_calculator",
|
||||
|
||||
@@ -22,9 +22,7 @@ input_stream: "IMAGE:image"
|
||||
input_stream: "ROI:roi"
|
||||
|
||||
# Pose landmarks within the given ROI. (NormalizedLandmarkList)
|
||||
# We have 25 (upper-body) landmarks
|
||||
# (see pose_landmark_upper_body_topology.svg), and there are other auxiliary key
|
||||
# points.
|
||||
# We have 25 (upper-body) landmarks (see pose_landmark_upper_body_topology.svg).
|
||||
# 0 - nose
|
||||
# 1 - right eye (inner)
|
||||
# 2 - right eye
|
||||
@@ -56,58 +54,39 @@ input_stream: "ROI:roi"
|
||||
# the MediaPipe framework will internally inform the downstream calculators of
|
||||
# the absence of this packet so that they don't wait for it unnecessarily.
|
||||
output_stream: "LANDMARKS:landmarks"
|
||||
# Auxiliary landmarks for deriving the ROI in the subsequent image.
|
||||
# (NormalizedLandmarkList)
|
||||
output_stream: "AUXILIARY_LANDMARKS:auxiliary_landmarks"
|
||||
|
||||
# Crops the rectangle that contains a pose from the input image.
|
||||
node {
|
||||
calculator: "ImageCroppingCalculator"
|
||||
# Transforms the input image into a 256x256 tensor while keeping the aspect
|
||||
# ratio (what is expected by the corresponding model), resulting in potential
|
||||
# letterboxing in the transformed image.
|
||||
node: {
|
||||
calculator: "ImageToTensorCalculator"
|
||||
input_stream: "IMAGE:image"
|
||||
input_stream: "NORM_RECT:roi"
|
||||
output_stream: "IMAGE:pose_region"
|
||||
options: {
|
||||
[mediapipe.ImageCroppingCalculatorOptions.ext] {
|
||||
border_mode: BORDER_REPLICATE
|
||||
output_max_width: 256
|
||||
output_max_height: 256
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Transforms the input image on CPU to a 256x256 image. To scale the input
|
||||
# image, the scale_mode option is set to FIT to preserve the aspect ratio,
|
||||
# resulting in potential letterboxing in the transformed image.
|
||||
node: {
|
||||
calculator: "ImageTransformationCalculator"
|
||||
input_stream: "IMAGE:pose_region"
|
||||
output_stream: "IMAGE:transformed_pose_region"
|
||||
output_stream: "TENSORS:input_tensors"
|
||||
output_stream: "LETTERBOX_PADDING:letterbox_padding"
|
||||
options: {
|
||||
[mediapipe.ImageTransformationCalculatorOptions.ext] {
|
||||
output_width: 256
|
||||
output_height: 256
|
||||
scale_mode: FIT
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Converts the transformed input image on CPU into a tensor.
|
||||
node {
|
||||
calculator: "TfLiteConverterCalculator"
|
||||
input_stream: "IMAGE:transformed_pose_region"
|
||||
output_stream: "TENSORS:input_tensors"
|
||||
options: {
|
||||
[mediapipe.TfLiteConverterCalculatorOptions.ext] {
|
||||
zero_center: false
|
||||
[mediapipe.ImageToTensorCalculatorOptions.ext] {
|
||||
output_tensor_width: 256
|
||||
output_tensor_height: 256
|
||||
keep_aspect_ratio: true
|
||||
output_tensor_float_range {
|
||||
min: 0.0
|
||||
max: 1.0
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Runs a TensorFlow Lite model inference on CPU.
|
||||
node {
|
||||
calculator: "TfLiteInferenceCalculator"
|
||||
calculator: "InferenceCalculator"
|
||||
input_stream: "TENSORS:input_tensors"
|
||||
output_stream: "TENSORS:output_tensors"
|
||||
options: {
|
||||
[mediapipe.TfLiteInferenceCalculatorOptions.ext] {
|
||||
[mediapipe.InferenceCalculatorOptions.ext] {
|
||||
model_path: "mediapipe/modules/pose_landmark/pose_landmark_upper_body.tflite"
|
||||
delegate { xnnpack {} }
|
||||
}
|
||||
@@ -117,7 +96,7 @@ node {
|
||||
# Splits a vector of TFLite tensors to multiple vectors according to the ranges
|
||||
# specified in option.
|
||||
node {
|
||||
calculator: "SplitTfLiteTensorVectorCalculator"
|
||||
calculator: "SplitTensorVectorCalculator"
|
||||
input_stream: "output_tensors"
|
||||
output_stream: "landmark_tensors"
|
||||
output_stream: "pose_flag_tensor"
|
||||
@@ -132,7 +111,7 @@ node {
|
||||
# Converts the pose-flag tensor into a float that represents the confidence
|
||||
# score of pose presence.
|
||||
node {
|
||||
calculator: "TfLiteTensorsToFloatsCalculator"
|
||||
calculator: "TensorsToFloatsCalculator"
|
||||
input_stream: "TENSORS:pose_flag_tensor"
|
||||
output_stream: "FLOAT:pose_presence_score"
|
||||
}
|
||||
@@ -150,7 +129,7 @@ node {
|
||||
}
|
||||
}
|
||||
|
||||
# Drop landmarks tensors if pose is not present.
|
||||
# Drops landmark tensors if pose is not present.
|
||||
node {
|
||||
calculator: "GateCalculator"
|
||||
input_stream: "landmark_tensors"
|
||||
@@ -158,15 +137,15 @@ node {
|
||||
output_stream: "ensured_landmark_tensors"
|
||||
}
|
||||
|
||||
# Decodes the landmark tensors into a vector of lanmarks, where the landmark
|
||||
# Decodes the landmark tensors into a vector of landmarks, where the landmark
|
||||
# coordinates are normalized by the size of the input image to the model.
|
||||
node {
|
||||
calculator: "TfLiteTensorsToLandmarksCalculator"
|
||||
calculator: "TensorsToLandmarksCalculator"
|
||||
input_stream: "TENSORS:ensured_landmark_tensors"
|
||||
output_stream: "NORM_LANDMARKS:raw_landmarks"
|
||||
options: {
|
||||
[mediapipe.TfLiteTensorsToLandmarksCalculatorOptions.ext] {
|
||||
num_landmarks: 31
|
||||
[mediapipe.TensorsToLandmarksCalculatorOptions.ext] {
|
||||
num_landmarks: 27
|
||||
input_image_width: 256
|
||||
input_image_height: 256
|
||||
}
|
||||
@@ -190,5 +169,20 @@ node {
|
||||
calculator: "LandmarkProjectionCalculator"
|
||||
input_stream: "NORM_LANDMARKS:adjusted_landmarks"
|
||||
input_stream: "NORM_RECT:roi"
|
||||
output_stream: "NORM_LANDMARKS:landmarks"
|
||||
output_stream: "NORM_LANDMARKS:all_landmarks"
|
||||
}
|
||||
|
||||
# Splits the landmarks into two sets: the actual pose landmarks and the
|
||||
# auxiliary landmarks.
|
||||
node {
|
||||
calculator: "SplitNormalizedLandmarkListCalculator"
|
||||
input_stream: "all_landmarks"
|
||||
output_stream: "landmarks"
|
||||
output_stream: "auxiliary_landmarks"
|
||||
options: {
|
||||
[mediapipe.SplitVectorCalculatorOptions.ext] {
|
||||
ranges: { begin: 0 end: 25 }
|
||||
ranges: { begin: 25 end: 27 }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -22,9 +22,7 @@ input_stream: "IMAGE:image"
|
||||
input_stream: "ROI:roi"
|
||||
|
||||
# Pose landmarks within the given ROI. (NormalizedLandmarkList)
|
||||
# We have 25 (upper-body) landmarks
|
||||
# (see pose_landmark_upper_body_topology.svg), and there are other auxiliary key
|
||||
# points.
|
||||
# We have 25 (upper-body) landmarks (see pose_landmark_upper_body_topology.svg).
|
||||
# 0 - nose
|
||||
# 1 - right eye (inner)
|
||||
# 2 - right eye
|
||||
@@ -56,58 +54,40 @@ input_stream: "ROI:roi"
|
||||
# the MediaPipe framework will internally inform the downstream calculators of
|
||||
# the absence of this packet so that they don't wait for it unnecessarily.
|
||||
output_stream: "LANDMARKS:landmarks"
|
||||
# Auxiliary landmarks for deriving the ROI in the subsequent image.
|
||||
# (NormalizedLandmarkList)
|
||||
output_stream: "AUXILIARY_LANDMARKS:auxiliary_landmarks"
|
||||
|
||||
# Crops the rectangle that contains a pose from the input image.
|
||||
node {
|
||||
calculator: "ImageCroppingCalculator"
|
||||
# Transforms the input image into a 256x256 tensor while keeping the aspect
|
||||
# ratio (what is expected by the corresponding model), resulting in potential
|
||||
# letterboxing in the transformed image.
|
||||
node: {
|
||||
calculator: "ImageToTensorCalculator"
|
||||
input_stream: "IMAGE_GPU:image"
|
||||
input_stream: "NORM_RECT:roi"
|
||||
output_stream: "IMAGE_GPU:pose_region"
|
||||
options: {
|
||||
[mediapipe.ImageCroppingCalculatorOptions.ext] {
|
||||
border_mode: BORDER_REPLICATE
|
||||
output_max_width: 256
|
||||
output_max_height: 256
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Transforms the input image on GPU to a 256x256 image. To scale the input
|
||||
# image, the scale_mode option is set to FIT to preserve the aspect ratio,
|
||||
# resulting in potential letterboxing in the transformed image.
|
||||
node: {
|
||||
calculator: "ImageTransformationCalculator"
|
||||
input_stream: "IMAGE_GPU:pose_region"
|
||||
output_stream: "IMAGE_GPU:transformed_pose_region"
|
||||
output_stream: "TENSORS:input_tensors"
|
||||
output_stream: "LETTERBOX_PADDING:letterbox_padding"
|
||||
options: {
|
||||
[mediapipe.ImageTransformationCalculatorOptions.ext] {
|
||||
output_width: 256
|
||||
output_height: 256
|
||||
scale_mode: FIT
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Converts the transformed input image on GPU into a tensor.
|
||||
node {
|
||||
calculator: "TfLiteConverterCalculator"
|
||||
input_stream: "IMAGE_GPU:transformed_pose_region"
|
||||
output_stream: "TENSORS_GPU:input_tensors"
|
||||
options: {
|
||||
[mediapipe.TfLiteConverterCalculatorOptions.ext] {
|
||||
zero_center: false
|
||||
[mediapipe.ImageToTensorCalculatorOptions.ext] {
|
||||
output_tensor_width: 256
|
||||
output_tensor_height: 256
|
||||
keep_aspect_ratio: true
|
||||
output_tensor_float_range {
|
||||
min: 0.0
|
||||
max: 1.0
|
||||
}
|
||||
gpu_origin: TOP_LEFT
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Runs a TensorFlow Lite model inference on GPU.
|
||||
node {
|
||||
calculator: "TfLiteInferenceCalculator"
|
||||
input_stream: "TENSORS_GPU:input_tensors"
|
||||
calculator: "InferenceCalculator"
|
||||
input_stream: "TENSORS:input_tensors"
|
||||
output_stream: "TENSORS:output_tensors"
|
||||
options: {
|
||||
[mediapipe.TfLiteInferenceCalculatorOptions.ext] {
|
||||
[mediapipe.InferenceCalculatorOptions.ext] {
|
||||
model_path: "mediapipe/modules/pose_landmark/pose_landmark_upper_body.tflite"
|
||||
}
|
||||
}
|
||||
@@ -116,7 +96,7 @@ node {
|
||||
# Splits a vector of TFLite tensors to multiple vectors according to the ranges
|
||||
# specified in option.
|
||||
node {
|
||||
calculator: "SplitTfLiteTensorVectorCalculator"
|
||||
calculator: "SplitTensorVectorCalculator"
|
||||
input_stream: "output_tensors"
|
||||
output_stream: "landmark_tensors"
|
||||
output_stream: "pose_flag_tensor"
|
||||
@@ -131,7 +111,7 @@ node {
|
||||
# Converts the pose-flag tensor into a float that represents the confidence
|
||||
# score of pose presence.
|
||||
node {
|
||||
calculator: "TfLiteTensorsToFloatsCalculator"
|
||||
calculator: "TensorsToFloatsCalculator"
|
||||
input_stream: "TENSORS:pose_flag_tensor"
|
||||
output_stream: "FLOAT:pose_presence_score"
|
||||
}
|
||||
@@ -149,7 +129,7 @@ node {
|
||||
}
|
||||
}
|
||||
|
||||
# Drop landmarks tensors if pose is not present.
|
||||
# Drops landmark tensors if pose is not present.
|
||||
node {
|
||||
calculator: "GateCalculator"
|
||||
input_stream: "landmark_tensors"
|
||||
@@ -157,15 +137,15 @@ node {
|
||||
output_stream: "ensured_landmark_tensors"
|
||||
}
|
||||
|
||||
# Decodes the landmark tensors into a vector of lanmarks, where the landmark
|
||||
# Decodes the landmark tensors into a vector of landmarks, where the landmark
|
||||
# coordinates are normalized by the size of the input image to the model.
|
||||
node {
|
||||
calculator: "TfLiteTensorsToLandmarksCalculator"
|
||||
calculator: "TensorsToLandmarksCalculator"
|
||||
input_stream: "TENSORS:ensured_landmark_tensors"
|
||||
output_stream: "NORM_LANDMARKS:raw_landmarks"
|
||||
options: {
|
||||
[mediapipe.TfLiteTensorsToLandmarksCalculatorOptions.ext] {
|
||||
num_landmarks: 31
|
||||
[mediapipe.TensorsToLandmarksCalculatorOptions.ext] {
|
||||
num_landmarks: 27
|
||||
input_image_width: 256
|
||||
input_image_height: 256
|
||||
}
|
||||
@@ -189,5 +169,20 @@ node {
|
||||
calculator: "LandmarkProjectionCalculator"
|
||||
input_stream: "NORM_LANDMARKS:adjusted_landmarks"
|
||||
input_stream: "NORM_RECT:roi"
|
||||
output_stream: "NORM_LANDMARKS:landmarks"
|
||||
output_stream: "NORM_LANDMARKS:all_landmarks"
|
||||
}
|
||||
|
||||
# Splits the landmarks into two sets: the actual pose landmarks and the
|
||||
# auxiliary landmarks.
|
||||
node {
|
||||
calculator: "SplitNormalizedLandmarkListCalculator"
|
||||
input_stream: "all_landmarks"
|
||||
output_stream: "landmarks"
|
||||
output_stream: "auxiliary_landmarks"
|
||||
options: {
|
||||
[mediapipe.SplitVectorCalculatorOptions.ext] {
|
||||
ranges: { begin: 0 end: 25 }
|
||||
ranges: { begin: 25 end: 27 }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -22,10 +22,12 @@ type: "PoseLandmarkUpperBodyCpu"
|
||||
# CPU image. (ImageFrame)
|
||||
input_stream: "IMAGE:image"
|
||||
|
||||
# Whether pose detection can be skipped when pose regions can already be
|
||||
# approximated from pose landmarks on the previous frame.
|
||||
input_side_packet: "CAN_SKIP_DETECTION:can_skip_detection"
|
||||
|
||||
# Pose landmarks within the given ROI. (NormalizedLandmarkList)
|
||||
# We have 25 (upper-body) landmarks
|
||||
# (see pose_landmark_upper_body_topology.svg), and there are other auxiliary key
|
||||
# points.
|
||||
# We have 25 (upper-body) landmarks (see pose_landmark_upper_body_topology.svg).
|
||||
# 0 - nose
|
||||
# 1 - right eye (inner)
|
||||
# 2 - right eye
|
||||
@@ -97,13 +99,23 @@ node {
|
||||
}
|
||||
|
||||
# Drops the incoming image if PoseLandmarkUpperBodyByRoiCpu was able to identify
|
||||
# pose presence in the previous image. Otherwise, passes the incoming image
|
||||
# through to trigger a new round of pose detection in PoseDetectionCpu.
|
||||
# pose presence in the previous image and skipping pose detection is enabled.
|
||||
# Otherwise, passes the incoming image through to trigger a new round of pose
|
||||
# detection in PoseDetectionCpu.
|
||||
node {
|
||||
calculator: "LogicCalculator"
|
||||
options: {
|
||||
[mediapipe.LogicCalculatorOptions.ext] { op: AND }
|
||||
}
|
||||
input_side_packet: "can_skip_detection"
|
||||
input_stream: "prev_pose_rect_from_landmarks_is_present"
|
||||
output_stream: "skip_detection"
|
||||
}
|
||||
node {
|
||||
calculator: "GateCalculator"
|
||||
input_stream: "image"
|
||||
input_stream: "image_size"
|
||||
input_stream: "DISALLOW:prev_pose_rect_from_landmarks_is_present"
|
||||
input_stream: "DISALLOW:skip_detection"
|
||||
output_stream: "image_for_pose_detection"
|
||||
output_stream: "image_size_for_pose_detection"
|
||||
options: {
|
||||
@@ -158,13 +170,14 @@ node {
|
||||
input_stream: "IMAGE:image"
|
||||
input_stream: "ROI:pose_rect"
|
||||
output_stream: "LANDMARKS:pose_landmarks"
|
||||
output_stream: "AUXILIARY_LANDMARKS:auxiliary_landmarks"
|
||||
}
|
||||
|
||||
# Calculates region of interest based on pose landmarks, so that can be reused
|
||||
# for subsequent image.
|
||||
# Calculates region of interest based on the auxiliary landmarks, to be used in
|
||||
# the subsequent image.
|
||||
node {
|
||||
calculator: "PoseLandmarkUpperBodyLandmarksToRoi"
|
||||
input_stream: "LANDMARKS:pose_landmarks"
|
||||
input_stream: "LANDMARKS:auxiliary_landmarks"
|
||||
input_stream: "IMAGE_SIZE:image_size"
|
||||
output_stream: "ROI:pose_rect_from_landmarks"
|
||||
}
|
||||
|
||||
@@ -22,10 +22,12 @@ type: "PoseLandmarkUpperBodyGpu"
|
||||
# GPU image. (GpuBuffer)
|
||||
input_stream: "IMAGE:image"
|
||||
|
||||
# Whether pose detection can be skipped when pose regions can already be
|
||||
# approximated from pose landmarks on the previous frame.
|
||||
input_side_packet: "CAN_SKIP_DETECTION:can_skip_detection"
|
||||
|
||||
# Pose landmarks within the given ROI. (NormalizedLandmarkList)
|
||||
# We have 25 (upper-body) landmarks
|
||||
# (see pose_landmark_upper_body_topology.svg), and there are other auxiliary key
|
||||
# points.
|
||||
# We have 25 (upper-body) landmarks (see pose_landmark_upper_body_topology.svg).
|
||||
# 0 - nose
|
||||
# 1 - right eye (inner)
|
||||
# 2 - right eye
|
||||
@@ -97,13 +99,23 @@ node {
|
||||
}
|
||||
|
||||
# Drops the incoming image if PoseLandmarkUpperBodyByRoiGpu was able to identify
|
||||
# pose presence in the previous image. Otherwise, passes the incoming image
|
||||
# through to trigger a new round of pose detection in PoseDetectionGpu.
|
||||
# pose presence in the previous image and skipping pose detection is enabled.
|
||||
# Otherwise, passes the incoming image through to trigger a new round of pose
|
||||
# detection in PoseDetectionGpu.
|
||||
node {
|
||||
calculator: "LogicCalculator"
|
||||
options: {
|
||||
[mediapipe.LogicCalculatorOptions.ext] { op: AND }
|
||||
}
|
||||
input_side_packet: "can_skip_detection"
|
||||
input_stream: "prev_pose_rect_from_landmarks_is_present"
|
||||
output_stream: "skip_detection"
|
||||
}
|
||||
node {
|
||||
calculator: "GateCalculator"
|
||||
input_stream: "image"
|
||||
input_stream: "image_size"
|
||||
input_stream: "DISALLOW:prev_pose_rect_from_landmarks_is_present"
|
||||
input_stream: "DISALLOW:skip_detection"
|
||||
output_stream: "image_for_pose_detection"
|
||||
output_stream: "image_size_for_pose_detection"
|
||||
options: {
|
||||
@@ -158,13 +170,14 @@ node {
|
||||
input_stream: "IMAGE:image"
|
||||
input_stream: "ROI:pose_rect"
|
||||
output_stream: "LANDMARKS:pose_landmarks"
|
||||
output_stream: "AUXILIARY_LANDMARKS:auxiliary_landmarks"
|
||||
}
|
||||
|
||||
# Calculates region of interest based on pose landmarks, so that can be reused
|
||||
# for subsequent image.
|
||||
# Calculates region of interest based on the auxiliary landmarks, to be used in
|
||||
# the subsequent image.
|
||||
node {
|
||||
calculator: "PoseLandmarkUpperBodyLandmarksToRoi"
|
||||
input_stream: "LANDMARKS:pose_landmarks"
|
||||
input_stream: "LANDMARKS:auxiliary_landmarks"
|
||||
input_stream: "IMAGE_SIZE:image_size"
|
||||
output_stream: "ROI:pose_rect_from_landmarks"
|
||||
}
|
||||
|
||||
@@ -13,21 +13,10 @@ input_stream: "IMAGE_SIZE:image_size"
|
||||
# ROI according to landmarks. (NormalizedRect)
|
||||
output_stream: "ROI:roi"
|
||||
|
||||
node {
|
||||
calculator: "SplitNormalizedLandmarkListCalculator"
|
||||
input_stream: "landmarks"
|
||||
output_stream: "alignment_landmarks"
|
||||
options: {
|
||||
[mediapipe.SplitVectorCalculatorOptions.ext] {
|
||||
ranges: { begin: 25 end: 27 }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Converts landmarks to a detection that tightly encloses all landmarks.
|
||||
node {
|
||||
calculator: "LandmarksToDetectionCalculator"
|
||||
input_stream: "NORM_LANDMARKS:alignment_landmarks"
|
||||
input_stream: "NORM_LANDMARKS:landmarks"
|
||||
output_stream: "DETECTION:detection"
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
# Experimental: Adds additional temporal filtering of the landmarks as a post
|
||||
# processing step to reduce jitter.
|
||||
#
|
||||
# MediaPipe graph to detect/predict pose landmarks. (CPU input, and inference is
|
||||
# executed on CPU.) This graph tries to skip pose detection as much as possible
|
||||
# by using previously detected/predicted landmarks for new images.
|
||||
#
|
||||
# It is required that "pose_detection.tflite" is available at
|
||||
# "mediapipe/modules/pose_detection/pose_detection.tflite"
|
||||
# path during execution.
|
||||
#
|
||||
# It is required that "pose_landmark_upper_body.tflite" is available at
|
||||
# "mediapipe/modules/pose_landmark/pose_landmark_upper_body.tflite"
|
||||
# path during execution.
|
||||
#
|
||||
# EXAMPLE:
|
||||
# node {
|
||||
# calculator: "PoseLandmarkUpperBodySmoothedCpu"
|
||||
# input_stream: "IMAGE:image"
|
||||
# output_stream: "LANDMARKS:pose_landmarks"
|
||||
# }
|
||||
|
||||
type: "PoseLandmarkUpperBodySmoothedCpu"
|
||||
|
||||
# CPU image. (ImageFrame)
|
||||
input_stream: "IMAGE:image"
|
||||
|
||||
# Whether pose detection can be skipped when pose regions can already be
|
||||
# approximated from pose landmarks on the previous frame.
|
||||
input_side_packet: "CAN_SKIP_DETECTION:can_skip_detection"
|
||||
|
||||
# The pose landmarks within the given ROI. (NormalizedLandmarkList)
|
||||
# We have 25 (upper-body) landmarks
|
||||
# (see pose_landmark_upper_body_topology.svg), and there are other auxiliary key
|
||||
# points.
|
||||
# 0 - nose
|
||||
# 1 - right eye (inner)
|
||||
# 2 - right eye
|
||||
# 3 - right eye (outer)
|
||||
# 4 - left eye (inner)
|
||||
# 5 - left eye
|
||||
# 6 - left eye (outer)
|
||||
# 7 - right ear
|
||||
# 8 - left ear
|
||||
# 9 - mouth (right)
|
||||
# 10 - mouth (left)
|
||||
# 11 - right shoulder
|
||||
# 12 - left shoulder
|
||||
# 13 - right elbow
|
||||
# 14 - left elbow
|
||||
# 15 - right wrist
|
||||
# 16 - left wrist
|
||||
# 17 - right pinky
|
||||
# 18 - left pinky
|
||||
# 19 - right index
|
||||
# 20 - left index
|
||||
# 21 - right thumb
|
||||
# 22 - left thumb
|
||||
# 23 - right hip
|
||||
# 24 - left hip
|
||||
#
|
||||
# NOTE: if a pose is not present within the given ROI, for this particular
|
||||
# timestamp there will not be an output packet in the LANDMARKS stream. However,
|
||||
# the MediaPipe framework will internally inform the downstream calculators of
|
||||
# the absence of this packet so that they don't wait for it unnecessarily.
|
||||
output_stream: "LANDMARKS:pose_landmarks"
|
||||
|
||||
# Extra outputs (for debugging, for instance).
|
||||
# Detected poses. (Detection)
|
||||
output_stream: "DETECTION:pose_detection"
|
||||
# Regions of interest calculated based on landmarks. (NormalizedRect)
|
||||
output_stream: "ROI_FROM_LANDMARKS:pose_rect_from_landmarks"
|
||||
# Regions of interest calculated based on pose detections. (NormalizedRect)
|
||||
output_stream: "ROI_FROM_DETECTION:pose_rect_from_detection"
|
||||
|
||||
# Subgraph that detects poses and corresponding landmarks.
|
||||
node {
|
||||
calculator: "PoseLandmarkUpperBodyCpu"
|
||||
input_stream: "IMAGE:image"
|
||||
output_stream: "LANDMARKS:unsmoothed_pose_landmarks"
|
||||
output_stream: "DETECTION:pose_detection"
|
||||
output_stream: "ROI_FROM_LANDMARKS:pose_rect_from_landmarks"
|
||||
output_stream: "ROI_FROM_DETECTION:pose_rect_from_detection"
|
||||
input_side_packet: "CAN_SKIP_DETECTION:can_skip_detection"
|
||||
}
|
||||
|
||||
# Calculates size of the image.
|
||||
node {
|
||||
calculator: "ImagePropertiesCalculator"
|
||||
input_stream: "IMAGE:image"
|
||||
output_stream: "SIZE:image_size"
|
||||
}
|
||||
|
||||
# Smoothes pose landmarks in order to reduce jitter.
|
||||
node {
|
||||
calculator: "LandmarksSmoothingCalculator"
|
||||
input_stream: "NORM_LANDMARKS:unsmoothed_pose_landmarks"
|
||||
input_stream: "IMAGE_SIZE:image_size"
|
||||
output_stream: "NORM_FILTERED_LANDMARKS:pose_landmarks"
|
||||
options: {
|
||||
[mediapipe.LandmarksSmoothingCalculatorOptions.ext] {
|
||||
velocity_filter: {
|
||||
window_size: 5
|
||||
velocity_scale: 10.0
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
# Experimental: Adds additional temporal filtering of the landmarks as a post
|
||||
# processing step to reduce jitter.
|
||||
#
|
||||
# MediaPipe graph to detect/predict pose landmarks. (GPU input, and inference is
|
||||
# executed on GPU.) This graph tries to skip pose detection as much as possible
|
||||
# by using previously detected/predicted landmarks for new images.
|
||||
#
|
||||
# It is required that "pose_detection.tflite" is available at
|
||||
# "mediapipe/modules/pose_detection/pose_detection.tflite"
|
||||
# path during execution.
|
||||
#
|
||||
# It is required that "pose_landmark_upper_body.tflite" is available at
|
||||
# "mediapipe/modules/pose_landmark/pose_landmark_upper_body.tflite"
|
||||
# path during execution.
|
||||
#
|
||||
# EXAMPLE:
|
||||
# node {
|
||||
# calculator: "PoseLandmarkUpperBodySmoothedGpu"
|
||||
# input_stream: "IMAGE:image"
|
||||
# output_stream: "LANDMARKS:pose_landmarks"
|
||||
# }
|
||||
|
||||
type: "PoseLandmarkUpperBodySmoothedGpu"
|
||||
|
||||
# GPU image. (GpuBuffer)
|
||||
input_stream: "IMAGE:image"
|
||||
|
||||
# Whether pose detection can be skipped when pose regions can already be
|
||||
# approximated from pose landmarks on the previous frame.
|
||||
input_side_packet: "CAN_SKIP_DETECTION:can_skip_detection"
|
||||
|
||||
# Pose landmarks within the given ROI. (NormalizedLandmarkList)
|
||||
# We have 25 (upper-body) landmarks
|
||||
# (see pose_landmark_upper_body_topology.svg), and there are other auxiliary key
|
||||
# points.
|
||||
# 0 - nose
|
||||
# 1 - right eye (inner)
|
||||
# 2 - right eye
|
||||
# 3 - right eye (outer)
|
||||
# 4 - left eye (inner)
|
||||
# 5 - left eye
|
||||
# 6 - left eye (outer)
|
||||
# 7 - right ear
|
||||
# 8 - left ear
|
||||
# 9 - mouth (right)
|
||||
# 10 - mouth (left)
|
||||
# 11 - right shoulder
|
||||
# 12 - left shoulder
|
||||
# 13 - right elbow
|
||||
# 14 - left elbow
|
||||
# 15 - right wrist
|
||||
# 16 - left wrist
|
||||
# 17 - right pinky
|
||||
# 18 - left pinky
|
||||
# 19 - right index
|
||||
# 20 - left index
|
||||
# 21 - right thumb
|
||||
# 22 - left thumb
|
||||
# 23 - right hip
|
||||
# 24 - left hip
|
||||
#
|
||||
# NOTE: if a pose is not present within the given ROI, for this particular
|
||||
# timestamp there will not be an output packet in the LANDMARKS stream. However,
|
||||
# the MediaPipe framework will internally inform the downstream calculators of
|
||||
# the absence of this packet so that they don't wait for it unnecessarily.
|
||||
output_stream: "LANDMARKS:pose_landmarks"
|
||||
|
||||
# Extra outputs (for debugging, for instance).
|
||||
# Detected poses. (Detection)
|
||||
output_stream: "DETECTION:pose_detection"
|
||||
# Regions of interest calculated based on landmarks. (NormalizedRect)
|
||||
output_stream: "ROI_FROM_LANDMARKS:pose_rect_from_landmarks"
|
||||
# Regions of interest calculated based on pose detections. (NormalizedRect)
|
||||
output_stream: "ROI_FROM_DETECTION:pose_rect_from_detection"
|
||||
|
||||
# Subgraph that detects poses and corresponding landmarks.
|
||||
node {
|
||||
calculator: "PoseLandmarkUpperBodyGpu"
|
||||
input_stream: "IMAGE:image"
|
||||
output_stream: "LANDMARKS:unsmoothed_pose_landmarks"
|
||||
output_stream: "DETECTION:pose_detection"
|
||||
output_stream: "ROI_FROM_LANDMARKS:pose_rect_from_landmarks"
|
||||
output_stream: "ROI_FROM_DETECTION:pose_rect_from_detection"
|
||||
input_side_packet: "CAN_SKIP_DETECTION:can_skip_detection"
|
||||
}
|
||||
|
||||
# Calculates size of the image.
|
||||
node {
|
||||
calculator: "ImagePropertiesCalculator"
|
||||
input_stream: "IMAGE_GPU:image"
|
||||
output_stream: "SIZE:image_size"
|
||||
}
|
||||
|
||||
# Smoothes pose landmarks in order to reduce jitter.
|
||||
node {
|
||||
calculator: "LandmarksSmoothingCalculator"
|
||||
input_stream: "NORM_LANDMARKS:unsmoothed_pose_landmarks"
|
||||
input_stream: "IMAGE_SIZE:image_size"
|
||||
output_stream: "NORM_FILTERED_LANDMARKS:pose_landmarks"
|
||||
options: {
|
||||
[mediapipe.LandmarksSmoothingCalculatorOptions.ext] {
|
||||
velocity_filter: {
|
||||
window_size: 5
|
||||
velocity_scale: 10.0
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user