Project import generated by Copybara.
GitOrigin-RevId: f7d09ed033907b893638a8eb4148efa11c0f09a6
This commit is contained in:
MediaPipe Team
@@ -20,35 +20,32 @@ cc_library(
|
||||
name = "mobile_calculators",
|
||||
deps = [
|
||||
"//mediapipe/calculators/core:flow_limiter_calculator",
|
||||
"//mediapipe/calculators/image:image_transformation_calculator",
|
||||
"//mediapipe/calculators/tflite:ssd_anchors_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_converter_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_inference_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_tensors_to_detections_calculator",
|
||||
"//mediapipe/calculators/util:annotation_overlay_calculator",
|
||||
"//mediapipe/calculators/util:detection_label_id_to_text_calculator",
|
||||
"//mediapipe/calculators/util:detection_letterbox_removal_calculator",
|
||||
"//mediapipe/calculators/util:detections_to_render_data_calculator",
|
||||
"//mediapipe/calculators/util:non_max_suppression_calculator",
|
||||
"//mediapipe/gpu:gpu_buffer_to_image_frame_calculator",
|
||||
"//mediapipe/gpu:image_frame_to_gpu_buffer_calculator",
|
||||
"//mediapipe/modules/face_detection:face_detection_front_cpu",
|
||||
"//mediapipe/modules/face_detection:face_detection_front_gpu",
|
||||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "desktop_tflite_calculators",
|
||||
name = "desktop_live_calculators",
|
||||
deps = [
|
||||
"//mediapipe/calculators/core:flow_limiter_calculator",
|
||||
"//mediapipe/calculators/image:image_transformation_calculator",
|
||||
"//mediapipe/calculators/tflite:ssd_anchors_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_converter_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_inference_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_tensors_to_detections_calculator",
|
||||
"//mediapipe/calculators/util:annotation_overlay_calculator",
|
||||
"//mediapipe/calculators/util:detection_label_id_to_text_calculator",
|
||||
"//mediapipe/calculators/util:detection_letterbox_removal_calculator",
|
||||
"//mediapipe/calculators/util:detections_to_render_data_calculator",
|
||||
"//mediapipe/calculators/util:non_max_suppression_calculator",
|
||||
"//mediapipe/modules/face_detection:face_detection_front_cpu",
|
||||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "desktop_live_gpu_calculators",
|
||||
deps = [
|
||||
"//mediapipe/calculators/core:flow_limiter_calculator",
|
||||
"//mediapipe/calculators/util:annotation_overlay_calculator",
|
||||
"//mediapipe/calculators/util:detections_to_render_data_calculator",
|
||||
"//mediapipe/modules/face_detection:face_detection_front_gpu",
|
||||
],
|
||||
)
|
||||
|
||||
@@ -58,15 +55,15 @@ load(
|
||||
)
|
||||
|
||||
mediapipe_binary_graph(
|
||||
name = "mobile_cpu_binary_graph",
|
||||
name = "face_detection_mobile_cpu_binary_graph",
|
||||
graph = "face_detection_mobile_cpu.pbtxt",
|
||||
output_name = "mobile_cpu.binarypb",
|
||||
output_name = "face_detection_mobile_cpu.binarypb",
|
||||
deps = [":mobile_calculators"],
|
||||
)
|
||||
|
||||
mediapipe_binary_graph(
|
||||
name = "mobile_gpu_binary_graph",
|
||||
name = "face_detection_mobile_gpu_binary_graph",
|
||||
graph = "face_detection_mobile_gpu.pbtxt",
|
||||
output_name = "mobile_gpu.binarypb",
|
||||
output_name = "face_detection_mobile_gpu.binarypb",
|
||||
deps = [":mobile_calculators"],
|
||||
)
|
||||
|
||||
@@ -1,28 +1,27 @@
|
||||
# MediaPipe graph that performs face detection with TensorFlow Lite on CPU.
|
||||
# Used in the examples in
|
||||
# mediapipe/examples/desktop/face_detection:face_detection_cpu.
|
||||
# MediaPipe graph that performs face mesh with TensorFlow Lite on CPU.
|
||||
|
||||
# Images on GPU coming into and out of the graph.
|
||||
# CPU buffer. (ImageFrame)
|
||||
input_stream: "input_video"
|
||||
|
||||
# Output image with rendered results. (ImageFrame)
|
||||
output_stream: "output_video"
|
||||
# Detected faces. (std::vector<Detection>)
|
||||
output_stream: "face_detections"
|
||||
|
||||
# Throttles the images flowing downstream for flow control. It passes through
|
||||
# the very first incoming image unaltered, and waits for
|
||||
# TfLiteTensorsToDetectionsCalculator downstream in the graph to finish
|
||||
# generating the corresponding detections before it passes through another
|
||||
# image. All images that come in while waiting are dropped, limiting the number
|
||||
# of in-flight images between this calculator and
|
||||
# TfLiteTensorsToDetectionsCalculator to 1. This prevents the nodes in between
|
||||
# from queuing up incoming images and data excessively, which leads to increased
|
||||
# latency and memory usage, unwanted in real-time mobile applications. It also
|
||||
# eliminates unnecessarily computation, e.g., a transformed image produced by
|
||||
# ImageTransformationCalculator may get dropped downstream if the subsequent
|
||||
# TfLiteConverterCalculator or TfLiteInferenceCalculator is still busy
|
||||
# processing previous inputs.
|
||||
# the very first incoming image unaltered, and waits for downstream nodes
|
||||
# (calculators and subgraphs) in the graph to finish their tasks before it
|
||||
# passes through another image. All images that come in while waiting are
|
||||
# dropped, limiting the number of in-flight images in most part of the graph to
|
||||
# 1. This prevents the downstream nodes from queuing up incoming images and data
|
||||
# excessively, which leads to increased latency and memory usage, unwanted in
|
||||
# real-time mobile applications. It also eliminates unnecessarily computation,
|
||||
# e.g., the output produced by a node may get dropped downstream if the
|
||||
# subsequent nodes are still busy processing previous inputs.
|
||||
node {
|
||||
calculator: "FlowLimiterCalculator"
|
||||
input_stream: "input_video"
|
||||
input_stream: "FINISHED:detections"
|
||||
input_stream: "FINISHED:output_video"
|
||||
input_stream_info: {
|
||||
tag_index: "FINISHED"
|
||||
back_edge: true
|
||||
@@ -30,141 +29,17 @@ node {
|
||||
output_stream: "throttled_input_video"
|
||||
}
|
||||
|
||||
# Transforms the input image on CPU to a 128x128 image. To scale the input
|
||||
# image, the scale_mode option is set to FIT to preserve the aspect ratio,
|
||||
# resulting in potential letterboxing in the transformed image.
|
||||
node: {
|
||||
calculator: "ImageTransformationCalculator"
|
||||
# Subgraph that detects faces.
|
||||
node {
|
||||
calculator: "FaceDetectionFrontCpu"
|
||||
input_stream: "IMAGE:throttled_input_video"
|
||||
output_stream: "IMAGE:transformed_input_video_cpu"
|
||||
output_stream: "LETTERBOX_PADDING:letterbox_padding"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.ImageTransformationCalculatorOptions] {
|
||||
output_width: 128
|
||||
output_height: 128
|
||||
scale_mode: FIT
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Converts the transformed input image on CPU into an image tensor stored as a
|
||||
# TfLiteTensor.
|
||||
node {
|
||||
calculator: "TfLiteConverterCalculator"
|
||||
input_stream: "IMAGE:transformed_input_video_cpu"
|
||||
output_stream: "TENSORS:image_tensor"
|
||||
}
|
||||
|
||||
# Runs a TensorFlow Lite model on CPU that takes an image tensor and outputs a
|
||||
# vector of tensors representing, for instance, detection boxes/keypoints and
|
||||
# scores.
|
||||
node {
|
||||
calculator: "TfLiteInferenceCalculator"
|
||||
input_stream: "TENSORS:image_tensor"
|
||||
output_stream: "TENSORS:detection_tensors"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.TfLiteInferenceCalculatorOptions] {
|
||||
model_path: "mediapipe/models/face_detection_front.tflite"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Generates a single side packet containing a vector of SSD anchors based on
|
||||
# the specification in the options.
|
||||
node {
|
||||
calculator: "SsdAnchorsCalculator"
|
||||
output_side_packet: "anchors"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.SsdAnchorsCalculatorOptions] {
|
||||
num_layers: 4
|
||||
min_scale: 0.1484375
|
||||
max_scale: 0.75
|
||||
input_size_height: 128
|
||||
input_size_width: 128
|
||||
anchor_offset_x: 0.5
|
||||
anchor_offset_y: 0.5
|
||||
strides: 8
|
||||
strides: 16
|
||||
strides: 16
|
||||
strides: 16
|
||||
aspect_ratios: 1.0
|
||||
fixed_anchor_size: true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Decodes the detection tensors generated by the TensorFlow Lite model, based on
|
||||
# the SSD anchors and the specification in the options, into a vector of
|
||||
# detections. Each detection describes a detected object.
|
||||
node {
|
||||
calculator: "TfLiteTensorsToDetectionsCalculator"
|
||||
input_stream: "TENSORS:detection_tensors"
|
||||
input_side_packet: "ANCHORS:anchors"
|
||||
output_stream: "DETECTIONS:detections"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.TfLiteTensorsToDetectionsCalculatorOptions] {
|
||||
num_classes: 1
|
||||
num_boxes: 896
|
||||
num_coords: 16
|
||||
box_coord_offset: 0
|
||||
keypoint_coord_offset: 4
|
||||
num_keypoints: 6
|
||||
num_values_per_keypoint: 2
|
||||
sigmoid_score: true
|
||||
score_clipping_thresh: 100.0
|
||||
reverse_output_order: true
|
||||
x_scale: 128.0
|
||||
y_scale: 128.0
|
||||
h_scale: 128.0
|
||||
w_scale: 128.0
|
||||
min_score_thresh: 0.5
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Performs non-max suppression to remove excessive detections.
|
||||
node {
|
||||
calculator: "NonMaxSuppressionCalculator"
|
||||
input_stream: "detections"
|
||||
output_stream: "filtered_detections"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.NonMaxSuppressionCalculatorOptions] {
|
||||
min_suppression_threshold: 0.3
|
||||
overlap_type: INTERSECTION_OVER_UNION
|
||||
algorithm: WEIGHTED
|
||||
return_empty_detections: true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Maps detection label IDs to the corresponding label text ("Face"). The label
|
||||
# map is provided in the label_map_path option.
|
||||
node {
|
||||
calculator: "DetectionLabelIdToTextCalculator"
|
||||
input_stream: "filtered_detections"
|
||||
output_stream: "labeled_detections"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.DetectionLabelIdToTextCalculatorOptions] {
|
||||
label_map_path: "mediapipe/models/face_detection_front_labelmap.txt"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Adjusts detection locations (already normalized to [0.f, 1.f]) on the
|
||||
# letterboxed image (after image transformation with the FIT scale mode) to the
|
||||
# corresponding locations on the same image with the letterbox removed (the
|
||||
# input image to the graph before image transformation).
|
||||
node {
|
||||
calculator: "DetectionLetterboxRemovalCalculator"
|
||||
input_stream: "DETECTIONS:labeled_detections"
|
||||
input_stream: "LETTERBOX_PADDING:letterbox_padding"
|
||||
output_stream: "DETECTIONS:output_detections"
|
||||
output_stream: "DETECTIONS:face_detections"
|
||||
}
|
||||
|
||||
# Converts the detections to drawing primitives for annotation overlay.
|
||||
node {
|
||||
calculator: "DetectionsToRenderDataCalculator"
|
||||
input_stream: "DETECTIONS:output_detections"
|
||||
input_stream: "DETECTIONS:face_detections"
|
||||
output_stream: "RENDER_DATA:render_data"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.DetectionsToRenderDataCalculatorOptions] {
|
||||
@@ -181,4 +56,3 @@ node {
|
||||
input_stream: "render_data"
|
||||
output_stream: "IMAGE:output_video"
|
||||
}
|
||||
|
||||
|
||||
@@ -1,29 +1,27 @@
|
||||
# MediaPipe graph that performs face detection with TensorFlow Lite on CPU.
|
||||
# Used in the examples in
|
||||
# mediapipie/examples/android/src/java/com/mediapipe/apps/facedetectioncpu and
|
||||
# mediapipie/examples/ios/facedetectioncpu.
|
||||
# MediaPipe graph that performs face mesh with TensorFlow Lite on CPU.
|
||||
|
||||
# Images on GPU coming into and out of the graph.
|
||||
# GPU buffer. (GpuBuffer)
|
||||
input_stream: "input_video"
|
||||
|
||||
# Output image with rendered results. (GpuBuffer)
|
||||
output_stream: "output_video"
|
||||
# Detected faces. (std::vector<Detection>)
|
||||
output_stream: "face_detections"
|
||||
|
||||
# Throttles the images flowing downstream for flow control. It passes through
|
||||
# the very first incoming image unaltered, and waits for
|
||||
# TfLiteTensorsToDetectionsCalculator downstream in the graph to finish
|
||||
# generating the corresponding detections before it passes through another
|
||||
# image. All images that come in while waiting are dropped, limiting the number
|
||||
# of in-flight images between this calculator and
|
||||
# TfLiteTensorsToDetectionsCalculator to 1. This prevents the nodes in between
|
||||
# from queuing up incoming images and data excessively, which leads to increased
|
||||
# latency and memory usage, unwanted in real-time mobile applications. It also
|
||||
# eliminates unnecessarily computation, e.g., a transformed image produced by
|
||||
# ImageTransformationCalculator may get dropped downstream if the subsequent
|
||||
# TfLiteConverterCalculator or TfLiteInferenceCalculator is still busy
|
||||
# processing previous inputs.
|
||||
# the very first incoming image unaltered, and waits for downstream nodes
|
||||
# (calculators and subgraphs) in the graph to finish their tasks before it
|
||||
# passes through another image. All images that come in while waiting are
|
||||
# dropped, limiting the number of in-flight images in most part of the graph to
|
||||
# 1. This prevents the downstream nodes from queuing up incoming images and data
|
||||
# excessively, which leads to increased latency and memory usage, unwanted in
|
||||
# real-time mobile applications. It also eliminates unnecessarily computation,
|
||||
# e.g., the output produced by a node may get dropped downstream if the
|
||||
# subsequent nodes are still busy processing previous inputs.
|
||||
node {
|
||||
calculator: "FlowLimiterCalculator"
|
||||
input_stream: "input_video"
|
||||
input_stream: "FINISHED:detections"
|
||||
input_stream: "FINISHED:output_video"
|
||||
input_stream_info: {
|
||||
tag_index: "FINISHED"
|
||||
back_edge: true
|
||||
@@ -41,141 +39,17 @@ node: {
|
||||
output_stream: "input_video_cpu"
|
||||
}
|
||||
|
||||
# Transforms the input image on CPU to a 128x128 image. To scale the input
|
||||
# image, the scale_mode option is set to FIT to preserve the aspect ratio,
|
||||
# resulting in potential letterboxing in the transformed image.
|
||||
node: {
|
||||
calculator: "ImageTransformationCalculator"
|
||||
# Subgraph that detects faces.
|
||||
node {
|
||||
calculator: "FaceDetectionFrontCpu"
|
||||
input_stream: "IMAGE:input_video_cpu"
|
||||
output_stream: "IMAGE:transformed_input_video_cpu"
|
||||
output_stream: "LETTERBOX_PADDING:letterbox_padding"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.ImageTransformationCalculatorOptions] {
|
||||
output_width: 128
|
||||
output_height: 128
|
||||
scale_mode: FIT
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Converts the transformed input image on CPU into an image tensor stored as a
|
||||
# TfLiteTensor.
|
||||
node {
|
||||
calculator: "TfLiteConverterCalculator"
|
||||
input_stream: "IMAGE:transformed_input_video_cpu"
|
||||
output_stream: "TENSORS:image_tensor"
|
||||
}
|
||||
|
||||
# Runs a TensorFlow Lite model on CPU that takes an image tensor and outputs a
|
||||
# vector of tensors representing, for instance, detection boxes/keypoints and
|
||||
# scores.
|
||||
node {
|
||||
calculator: "TfLiteInferenceCalculator"
|
||||
input_stream: "TENSORS:image_tensor"
|
||||
output_stream: "TENSORS:detection_tensors"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.TfLiteInferenceCalculatorOptions] {
|
||||
model_path: "mediapipe/models/face_detection_front.tflite"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Generates a single side packet containing a vector of SSD anchors based on
|
||||
# the specification in the options.
|
||||
node {
|
||||
calculator: "SsdAnchorsCalculator"
|
||||
output_side_packet: "anchors"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.SsdAnchorsCalculatorOptions] {
|
||||
num_layers: 4
|
||||
min_scale: 0.1484375
|
||||
max_scale: 0.75
|
||||
input_size_height: 128
|
||||
input_size_width: 128
|
||||
anchor_offset_x: 0.5
|
||||
anchor_offset_y: 0.5
|
||||
strides: 8
|
||||
strides: 16
|
||||
strides: 16
|
||||
strides: 16
|
||||
aspect_ratios: 1.0
|
||||
fixed_anchor_size: true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Decodes the detection tensors generated by the TensorFlow Lite model, based on
|
||||
# the SSD anchors and the specification in the options, into a vector of
|
||||
# detections. Each detection describes a detected object.
|
||||
node {
|
||||
calculator: "TfLiteTensorsToDetectionsCalculator"
|
||||
input_stream: "TENSORS:detection_tensors"
|
||||
input_side_packet: "ANCHORS:anchors"
|
||||
output_stream: "DETECTIONS:detections"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.TfLiteTensorsToDetectionsCalculatorOptions] {
|
||||
num_classes: 1
|
||||
num_boxes: 896
|
||||
num_coords: 16
|
||||
box_coord_offset: 0
|
||||
keypoint_coord_offset: 4
|
||||
num_keypoints: 6
|
||||
num_values_per_keypoint: 2
|
||||
sigmoid_score: true
|
||||
score_clipping_thresh: 100.0
|
||||
reverse_output_order: true
|
||||
x_scale: 128.0
|
||||
y_scale: 128.0
|
||||
h_scale: 128.0
|
||||
w_scale: 128.0
|
||||
min_score_thresh: 0.5
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Performs non-max suppression to remove excessive detections.
|
||||
node {
|
||||
calculator: "NonMaxSuppressionCalculator"
|
||||
input_stream: "detections"
|
||||
output_stream: "filtered_detections"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.NonMaxSuppressionCalculatorOptions] {
|
||||
min_suppression_threshold: 0.3
|
||||
overlap_type: INTERSECTION_OVER_UNION
|
||||
algorithm: WEIGHTED
|
||||
return_empty_detections: true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Maps detection label IDs to the corresponding label text ("Face"). The label
|
||||
# map is provided in the label_map_path option.
|
||||
node {
|
||||
calculator: "DetectionLabelIdToTextCalculator"
|
||||
input_stream: "filtered_detections"
|
||||
output_stream: "labeled_detections"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.DetectionLabelIdToTextCalculatorOptions] {
|
||||
label_map_path: "mediapipe/models/face_detection_front_labelmap.txt"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Adjusts detection locations (already normalized to [0.f, 1.f]) on the
|
||||
# letterboxed image (after image transformation with the FIT scale mode) to the
|
||||
# corresponding locations on the same image with the letterbox removed (the
|
||||
# input image to the graph before image transformation).
|
||||
node {
|
||||
calculator: "DetectionLetterboxRemovalCalculator"
|
||||
input_stream: "DETECTIONS:labeled_detections"
|
||||
input_stream: "LETTERBOX_PADDING:letterbox_padding"
|
||||
output_stream: "DETECTIONS:output_detections"
|
||||
output_stream: "DETECTIONS:face_detections"
|
||||
}
|
||||
|
||||
# Converts the detections to drawing primitives for annotation overlay.
|
||||
node {
|
||||
calculator: "DetectionsToRenderDataCalculator"
|
||||
input_stream: "DETECTIONS:output_detections"
|
||||
input_stream: "DETECTIONS:face_detections"
|
||||
output_stream: "RENDER_DATA:render_data"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.DetectionsToRenderDataCalculatorOptions] {
|
||||
|
||||
@@ -1,29 +1,27 @@
|
||||
# MediaPipe graph that performs face detection with TensorFlow Lite on GPU.
|
||||
# Used in the examples in
|
||||
# mediapipie/examples/android/src/java/com/mediapipe/apps/facedetectiongpu and
|
||||
# mediapipie/examples/ios/facedetectiongpu.
|
||||
# MediaPipe graph that performs face mesh with TensorFlow Lite on GPU.
|
||||
|
||||
# Images on GPU coming into and out of the graph.
|
||||
# GPU buffer. (GpuBuffer)
|
||||
input_stream: "input_video"
|
||||
|
||||
# Output image with rendered results. (GpuBuffer)
|
||||
output_stream: "output_video"
|
||||
# Detected faces. (std::vector<Detection>)
|
||||
output_stream: "face_detections"
|
||||
|
||||
# Throttles the images flowing downstream for flow control. It passes through
|
||||
# the very first incoming image unaltered, and waits for
|
||||
# TfLiteTensorsToDetectionsCalculator downstream in the graph to finish
|
||||
# generating the corresponding detections before it passes through another
|
||||
# image. All images that come in while waiting are dropped, limiting the number
|
||||
# of in-flight images between this calculator and
|
||||
# TfLiteTensorsToDetectionsCalculator to 1. This prevents the nodes in between
|
||||
# from queuing up incoming images and data excessively, which leads to increased
|
||||
# latency and memory usage, unwanted in real-time mobile applications. It also
|
||||
# eliminates unnecessarily computation, e.g., a transformed image produced by
|
||||
# ImageTransformationCalculator may get dropped downstream if the subsequent
|
||||
# TfLiteConverterCalculator or TfLiteInferenceCalculator is still busy
|
||||
# processing previous inputs.
|
||||
# the very first incoming image unaltered, and waits for downstream nodes
|
||||
# (calculators and subgraphs) in the graph to finish their tasks before it
|
||||
# passes through another image. All images that come in while waiting are
|
||||
# dropped, limiting the number of in-flight images in most part of the graph to
|
||||
# 1. This prevents the downstream nodes from queuing up incoming images and data
|
||||
# excessively, which leads to increased latency and memory usage, unwanted in
|
||||
# real-time mobile applications. It also eliminates unnecessarily computation,
|
||||
# e.g., the output produced by a node may get dropped downstream if the
|
||||
# subsequent nodes are still busy processing previous inputs.
|
||||
node {
|
||||
calculator: "FlowLimiterCalculator"
|
||||
input_stream: "input_video"
|
||||
input_stream: "FINISHED:detections"
|
||||
input_stream: "FINISHED:output_video"
|
||||
input_stream_info: {
|
||||
tag_index: "FINISHED"
|
||||
back_edge: true
|
||||
@@ -31,141 +29,17 @@ node {
|
||||
output_stream: "throttled_input_video"
|
||||
}
|
||||
|
||||
# Transforms the input image on GPU to a 128x128 image. To scale the input
|
||||
# image, the scale_mode option is set to FIT to preserve the aspect ratio,
|
||||
# resulting in potential letterboxing in the transformed image.
|
||||
node: {
|
||||
calculator: "ImageTransformationCalculator"
|
||||
input_stream: "IMAGE_GPU:throttled_input_video"
|
||||
output_stream: "IMAGE_GPU:transformed_input_video"
|
||||
output_stream: "LETTERBOX_PADDING:letterbox_padding"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.ImageTransformationCalculatorOptions] {
|
||||
output_width: 128
|
||||
output_height: 128
|
||||
scale_mode: FIT
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Converts the transformed input image on GPU into an image tensor stored as a
|
||||
# TfLiteTensor.
|
||||
# Subgraph that detects faces.
|
||||
node {
|
||||
calculator: "TfLiteConverterCalculator"
|
||||
input_stream: "IMAGE_GPU:transformed_input_video"
|
||||
output_stream: "TENSORS_GPU:image_tensor"
|
||||
}
|
||||
|
||||
# Runs a TensorFlow Lite model on GPU that takes an image tensor and outputs a
|
||||
# vector of tensors representing, for instance, detection boxes/keypoints and
|
||||
# scores.
|
||||
node {
|
||||
calculator: "TfLiteInferenceCalculator"
|
||||
input_stream: "TENSORS_GPU:image_tensor"
|
||||
output_stream: "TENSORS_GPU:detection_tensors"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.TfLiteInferenceCalculatorOptions] {
|
||||
model_path: "mediapipe/models/face_detection_front.tflite"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Generates a single side packet containing a vector of SSD anchors based on
|
||||
# the specification in the options.
|
||||
node {
|
||||
calculator: "SsdAnchorsCalculator"
|
||||
output_side_packet: "anchors"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.SsdAnchorsCalculatorOptions] {
|
||||
num_layers: 4
|
||||
min_scale: 0.1484375
|
||||
max_scale: 0.75
|
||||
input_size_height: 128
|
||||
input_size_width: 128
|
||||
anchor_offset_x: 0.5
|
||||
anchor_offset_y: 0.5
|
||||
strides: 8
|
||||
strides: 16
|
||||
strides: 16
|
||||
strides: 16
|
||||
aspect_ratios: 1.0
|
||||
fixed_anchor_size: true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Decodes the detection tensors generated by the TensorFlow Lite model, based on
|
||||
# the SSD anchors and the specification in the options, into a vector of
|
||||
# detections. Each detection describes a detected object.
|
||||
node {
|
||||
calculator: "TfLiteTensorsToDetectionsCalculator"
|
||||
input_stream: "TENSORS_GPU:detection_tensors"
|
||||
input_side_packet: "ANCHORS:anchors"
|
||||
output_stream: "DETECTIONS:detections"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.TfLiteTensorsToDetectionsCalculatorOptions] {
|
||||
num_classes: 1
|
||||
num_boxes: 896
|
||||
num_coords: 16
|
||||
box_coord_offset: 0
|
||||
keypoint_coord_offset: 4
|
||||
num_keypoints: 6
|
||||
num_values_per_keypoint: 2
|
||||
sigmoid_score: true
|
||||
score_clipping_thresh: 100.0
|
||||
reverse_output_order: true
|
||||
x_scale: 128.0
|
||||
y_scale: 128.0
|
||||
h_scale: 128.0
|
||||
w_scale: 128.0
|
||||
min_score_thresh: 0.5
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Performs non-max suppression to remove excessive detections.
|
||||
node {
|
||||
calculator: "NonMaxSuppressionCalculator"
|
||||
input_stream: "detections"
|
||||
output_stream: "filtered_detections"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.NonMaxSuppressionCalculatorOptions] {
|
||||
min_suppression_threshold: 0.3
|
||||
overlap_type: INTERSECTION_OVER_UNION
|
||||
algorithm: WEIGHTED
|
||||
return_empty_detections: true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Maps detection label IDs to the corresponding label text ("Face"). The label
|
||||
# map is provided in the label_map_path option.
|
||||
node {
|
||||
calculator: "DetectionLabelIdToTextCalculator"
|
||||
input_stream: "filtered_detections"
|
||||
output_stream: "labeled_detections"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.DetectionLabelIdToTextCalculatorOptions] {
|
||||
label_map_path: "mediapipe/models/face_detection_front_labelmap.txt"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Adjusts detection locations (already normalized to [0.f, 1.f]) on the
|
||||
# letterboxed image (after image transformation with the FIT scale mode) to the
|
||||
# corresponding locations on the same image with the letterbox removed (the
|
||||
# input image to the graph before image transformation).
|
||||
node {
|
||||
calculator: "DetectionLetterboxRemovalCalculator"
|
||||
input_stream: "DETECTIONS:labeled_detections"
|
||||
input_stream: "LETTERBOX_PADDING:letterbox_padding"
|
||||
output_stream: "DETECTIONS:output_detections"
|
||||
calculator: "FaceDetectionFrontGpu"
|
||||
input_stream: "IMAGE:throttled_input_video"
|
||||
output_stream: "DETECTIONS:face_detections"
|
||||
}
|
||||
|
||||
# Converts the detections to drawing primitives for annotation overlay.
|
||||
node {
|
||||
calculator: "DetectionsToRenderDataCalculator"
|
||||
input_stream: "DETECTIONS:output_detections"
|
||||
input_stream: "DETECTIONS:face_detections"
|
||||
output_stream: "RENDER_DATA:render_data"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.DetectionsToRenderDataCalculatorOptions] {
|
||||
|
||||
@@ -42,10 +42,11 @@ cc_library(
|
||||
name = "desktop_tflite_calculators",
|
||||
deps = [
|
||||
":desktop_offline_calculators",
|
||||
"//mediapipe/calculators/core:constant_side_packet_calculator",
|
||||
"//mediapipe/calculators/core:merge_calculator",
|
||||
"//mediapipe/graphs/hand_tracking/subgraphs:hand_detection_cpu",
|
||||
"//mediapipe/graphs/hand_tracking/subgraphs:hand_landmark_cpu",
|
||||
"//mediapipe/graphs/hand_tracking/subgraphs:renderer_cpu",
|
||||
"//mediapipe/graphs/hand_tracking/subgraphs:hand_renderer_cpu",
|
||||
"//mediapipe/modules/hand_landmark:hand_landmark_tracking_cpu",
|
||||
"//mediapipe/modules/palm_detection:palm_detection_cpu",
|
||||
],
|
||||
)
|
||||
|
||||
@@ -59,13 +60,10 @@ mediapipe_binary_graph(
|
||||
cc_library(
|
||||
name = "mobile_calculators",
|
||||
deps = [
|
||||
"//mediapipe/calculators/core:constant_side_packet_calculator",
|
||||
"//mediapipe/calculators/core:flow_limiter_calculator",
|
||||
"//mediapipe/calculators/core:gate_calculator",
|
||||
"//mediapipe/calculators/core:merge_calculator",
|
||||
"//mediapipe/calculators/core:previous_loopback_calculator",
|
||||
"//mediapipe/graphs/hand_tracking/subgraphs:hand_detection_gpu",
|
||||
"//mediapipe/graphs/hand_tracking/subgraphs:hand_landmark_gpu",
|
||||
"//mediapipe/graphs/hand_tracking/subgraphs:renderer_gpu",
|
||||
"//mediapipe/graphs/hand_tracking/subgraphs:hand_renderer_gpu",
|
||||
"//mediapipe/modules/hand_landmark:hand_landmark_tracking_gpu",
|
||||
],
|
||||
)
|
||||
|
||||
@@ -76,52 +74,13 @@ mediapipe_binary_graph(
|
||||
deps = [":mobile_calculators"],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "multi_hand_desktop_tflite_calculators",
|
||||
deps = [
|
||||
":desktop_offline_calculators",
|
||||
"//mediapipe/calculators/util:association_norm_rect_calculator",
|
||||
"//mediapipe/calculators/util:collection_has_min_size_calculator",
|
||||
"//mediapipe/graphs/hand_tracking/subgraphs:multi_hand_detection_cpu",
|
||||
"//mediapipe/graphs/hand_tracking/subgraphs:multi_hand_landmark_cpu",
|
||||
"//mediapipe/graphs/hand_tracking/subgraphs:multi_hand_renderer_cpu",
|
||||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "multi_hand_mobile_calculators",
|
||||
deps = [
|
||||
"//mediapipe/calculators/core:flow_limiter_calculator",
|
||||
"//mediapipe/calculators/core:gate_calculator",
|
||||
"//mediapipe/calculators/core:previous_loopback_calculator",
|
||||
"//mediapipe/calculators/util:association_norm_rect_calculator",
|
||||
"//mediapipe/calculators/util:collection_has_min_size_calculator",
|
||||
"//mediapipe/graphs/hand_tracking/subgraphs:multi_hand_detection_gpu",
|
||||
"//mediapipe/graphs/hand_tracking/subgraphs:multi_hand_landmark_gpu",
|
||||
"//mediapipe/graphs/hand_tracking/subgraphs:multi_hand_renderer_gpu",
|
||||
],
|
||||
)
|
||||
|
||||
mediapipe_binary_graph(
|
||||
name = "multi_hand_tracking_desktop_live_binary_graph",
|
||||
graph = "multi_hand_tracking_desktop_live.pbtxt",
|
||||
output_name = "multi_hand_tracking_desktop_live.binarypb",
|
||||
deps = [":multi_hand_desktop_tflite_calculators"],
|
||||
)
|
||||
|
||||
mediapipe_binary_graph(
|
||||
name = "multi_hand_tracking_mobile_gpu_binary_graph",
|
||||
graph = "multi_hand_tracking_mobile.pbtxt",
|
||||
output_name = "multi_hand_tracking_mobile_gpu.binarypb",
|
||||
deps = [":multi_hand_mobile_calculators"],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "detection_mobile_calculators",
|
||||
deps = [
|
||||
"//mediapipe/calculators/core:flow_limiter_calculator",
|
||||
"//mediapipe/graphs/hand_tracking/subgraphs:hand_detection_gpu",
|
||||
"//mediapipe/graphs/hand_tracking/subgraphs:renderer_gpu",
|
||||
"//mediapipe/calculators/util:annotation_overlay_calculator",
|
||||
"//mediapipe/calculators/util:detections_to_render_data_calculator",
|
||||
"//mediapipe/modules/palm_detection:palm_detection_gpu",
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
@@ -15,19 +15,3 @@
|
||||
licenses(["notice"])
|
||||
|
||||
package(default_visibility = ["//visibility:public"])
|
||||
|
||||
cc_library(
|
||||
name = "hand_landmarks_to_rect_calculator",
|
||||
srcs = ["hand_landmarks_to_rect_calculator.cc"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:calculator_options_cc_proto",
|
||||
"//mediapipe/framework/formats:landmark_cc_proto",
|
||||
"//mediapipe/framework/formats:location_data_cc_proto",
|
||||
"//mediapipe/framework/formats:rect_cc_proto",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/port:status",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
@@ -1,167 +0,0 @@
|
||||
// Copyright 2020 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#include <cmath>
|
||||
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/calculator_options.pb.h"
|
||||
#include "mediapipe/framework/formats/landmark.pb.h"
|
||||
#include "mediapipe/framework/formats/rect.pb.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
namespace {
|
||||
|
||||
constexpr char kNormalizedLandmarksTag[] = "NORM_LANDMARKS";
|
||||
constexpr char kNormRectTag[] = "NORM_RECT";
|
||||
constexpr char kImageSizeTag[] = "IMAGE_SIZE";
|
||||
constexpr int kWristJoint = 0;
|
||||
constexpr int kMiddleFingerPIPJoint = 6;
|
||||
constexpr int kIndexFingerPIPJoint = 4;
|
||||
constexpr int kRingFingerPIPJoint = 8;
|
||||
constexpr float kTargetAngle = M_PI * 0.5f;
|
||||
|
||||
inline float NormalizeRadians(float angle) {
|
||||
return angle - 2 * M_PI * std::floor((angle - (-M_PI)) / (2 * M_PI));
|
||||
}
|
||||
|
||||
float ComputeRotation(const NormalizedLandmarkList& landmarks,
|
||||
const std::pair<int, int>& image_size) {
|
||||
const float x0 = landmarks.landmark(kWristJoint).x() * image_size.first;
|
||||
const float y0 = landmarks.landmark(kWristJoint).y() * image_size.second;
|
||||
|
||||
float x1 = (landmarks.landmark(kIndexFingerPIPJoint).x() +
|
||||
landmarks.landmark(kRingFingerPIPJoint).x()) /
|
||||
2.f;
|
||||
float y1 = (landmarks.landmark(kIndexFingerPIPJoint).y() +
|
||||
landmarks.landmark(kRingFingerPIPJoint).y()) /
|
||||
2.f;
|
||||
x1 = (x1 + landmarks.landmark(kMiddleFingerPIPJoint).x()) / 2.f *
|
||||
image_size.first;
|
||||
y1 = (y1 + landmarks.landmark(kMiddleFingerPIPJoint).y()) / 2.f *
|
||||
image_size.second;
|
||||
|
||||
const float rotation =
|
||||
NormalizeRadians(kTargetAngle - std::atan2(-(y1 - y0), x1 - x0));
|
||||
return rotation;
|
||||
}
|
||||
|
||||
::mediapipe::Status NormalizedLandmarkListToRect(
|
||||
const NormalizedLandmarkList& landmarks,
|
||||
const std::pair<int, int>& image_size, NormalizedRect* rect) {
|
||||
const float rotation = ComputeRotation(landmarks, image_size);
|
||||
const float reverse_angle = NormalizeRadians(-rotation);
|
||||
|
||||
// Find boundaries of landmarks.
|
||||
float max_x = std::numeric_limits<float>::min();
|
||||
float max_y = std::numeric_limits<float>::min();
|
||||
float min_x = std::numeric_limits<float>::max();
|
||||
float min_y = std::numeric_limits<float>::max();
|
||||
for (int i = 0; i < landmarks.landmark_size(); ++i) {
|
||||
max_x = std::max(max_x, landmarks.landmark(i).x());
|
||||
max_y = std::max(max_y, landmarks.landmark(i).y());
|
||||
min_x = std::min(min_x, landmarks.landmark(i).x());
|
||||
min_y = std::min(min_y, landmarks.landmark(i).y());
|
||||
}
|
||||
const float axis_aligned_center_x = (max_x + min_x) / 2.f;
|
||||
const float axis_aligned_center_y = (max_y + min_y) / 2.f;
|
||||
|
||||
// Find boundaries of rotated landmarks.
|
||||
max_x = std::numeric_limits<float>::min();
|
||||
max_y = std::numeric_limits<float>::min();
|
||||
min_x = std::numeric_limits<float>::max();
|
||||
min_y = std::numeric_limits<float>::max();
|
||||
for (int i = 0; i < landmarks.landmark_size(); ++i) {
|
||||
const float original_x =
|
||||
(landmarks.landmark(i).x() - axis_aligned_center_x) * image_size.first;
|
||||
const float original_y =
|
||||
(landmarks.landmark(i).y() - axis_aligned_center_y) * image_size.second;
|
||||
|
||||
const float projected_x = original_x * std::cos(reverse_angle) -
|
||||
original_y * std::sin(reverse_angle);
|
||||
const float projected_y = original_x * std::sin(reverse_angle) +
|
||||
original_y * std::cos(reverse_angle);
|
||||
|
||||
max_x = std::max(max_x, projected_x);
|
||||
max_y = std::max(max_y, projected_y);
|
||||
min_x = std::min(min_x, projected_x);
|
||||
min_y = std::min(min_y, projected_y);
|
||||
}
|
||||
const float projected_center_x = (max_x + min_x) / 2.f;
|
||||
const float projected_center_y = (max_y + min_y) / 2.f;
|
||||
|
||||
const float center_x = projected_center_x * std::cos(rotation) -
|
||||
projected_center_y * std::sin(rotation) +
|
||||
image_size.first * axis_aligned_center_x;
|
||||
const float center_y = projected_center_x * std::sin(rotation) +
|
||||
projected_center_y * std::cos(rotation) +
|
||||
image_size.second * axis_aligned_center_y;
|
||||
const float width = (max_x - min_x) / image_size.first;
|
||||
const float height = (max_y - min_y) / image_size.second;
|
||||
|
||||
rect->set_x_center(center_x / image_size.first);
|
||||
rect->set_y_center(center_y / image_size.second);
|
||||
rect->set_width(width);
|
||||
rect->set_height(height);
|
||||
rect->set_rotation(rotation);
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
// A calculator that converts subset of hand landmarks to a bounding box
|
||||
// NormalizedRect. The rotation angle of the bounding box is computed based on
|
||||
// 1) the wrist joint and 2) the average of PIP joints of index finger, middle
|
||||
// finger and ring finger. After rotation, the vector from the wrist to the mean
|
||||
// of PIP joints is expected to be vertical with wrist at the bottom and the
|
||||
// mean of PIP joints at the top.
|
||||
class HandLandmarksToRectCalculator : public CalculatorBase {
|
||||
public:
|
||||
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
|
||||
cc->Inputs().Tag(kNormalizedLandmarksTag).Set<NormalizedLandmarkList>();
|
||||
cc->Inputs().Tag(kImageSizeTag).Set<std::pair<int, int>>();
|
||||
cc->Outputs().Tag(kNormRectTag).Set<NormalizedRect>();
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status Open(CalculatorContext* cc) override {
|
||||
cc->SetOffset(TimestampDiff(0));
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status Process(CalculatorContext* cc) override {
|
||||
if (cc->Inputs().Tag(kNormalizedLandmarksTag).IsEmpty()) {
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
RET_CHECK(!cc->Inputs().Tag(kImageSizeTag).IsEmpty());
|
||||
|
||||
std::pair<int, int> image_size =
|
||||
cc->Inputs().Tag(kImageSizeTag).Get<std::pair<int, int>>();
|
||||
const auto& landmarks =
|
||||
cc->Inputs().Tag(kNormalizedLandmarksTag).Get<NormalizedLandmarkList>();
|
||||
auto output_rect = absl::make_unique<NormalizedRect>();
|
||||
MP_RETURN_IF_ERROR(
|
||||
NormalizedLandmarkListToRect(landmarks, image_size, output_rect.get()));
|
||||
cc->Outputs()
|
||||
.Tag(kNormRectTag)
|
||||
.Add(output_rect.release(), cc->InputTimestamp());
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
};
|
||||
REGISTER_CALCULATOR(HandLandmarksToRectCalculator);
|
||||
|
||||
} // namespace mediapipe
|
||||
@@ -16,11 +16,10 @@ node {
|
||||
output_stream: "VIDEO_PRESTREAM:input_video_header"
|
||||
}
|
||||
|
||||
# Performs hand detection model on the input frames. See
|
||||
# hand_detection_cpu.pbtxt for the detail of the sub-graph.
|
||||
# Detects palms.
|
||||
node {
|
||||
calculator: "HandDetectionSubgraph"
|
||||
input_stream: "input_video"
|
||||
calculator: "PalmDetectionCpu"
|
||||
input_stream: "IMAGE:input_video"
|
||||
output_stream: "DETECTIONS:output_detections"
|
||||
}
|
||||
|
||||
|
||||
@@ -3,15 +3,16 @@
|
||||
# Used in the example in
|
||||
# mediapipe/examples/desktop/hand_tracking:hand_detection_cpu.
|
||||
|
||||
# Images coming into and out of the graph.
|
||||
# CPU image. (ImageFrame)
|
||||
input_stream: "input_video"
|
||||
|
||||
# CPU image. (ImageFrame)
|
||||
output_stream: "output_video"
|
||||
|
||||
# Performs hand detection model on the input frames. See
|
||||
# hand_detection_cpu.pbtxt for the detail of the sub-graph.
|
||||
# Detects palms.
|
||||
node {
|
||||
calculator: "HandDetectionSubgraph"
|
||||
input_stream: "input_video"
|
||||
calculator: "PalmDetectionCpu"
|
||||
input_stream: "IMAGE:input_video"
|
||||
output_stream: "DETECTIONS:output_detections"
|
||||
}
|
||||
|
||||
|
||||
@@ -3,24 +3,26 @@
|
||||
# mediapipe/examples/android/src/java/com/mediapipe/apps/handdetectiongpu and
|
||||
# mediapipe/examples/ios/handdetectiongpu.
|
||||
|
||||
# Images coming into and out of the graph.
|
||||
# GPU image. (GpuBuffer)
|
||||
input_stream: "input_video"
|
||||
|
||||
# GPU image. (GpuBuffer)
|
||||
output_stream: "output_video"
|
||||
|
||||
# Throttles the images flowing downstream for flow control. It passes through
|
||||
# the very first incoming image unaltered, and waits for HandDetectionSubgraph
|
||||
# the very first incoming image unaltered, and waits for PalmDetectionGpu
|
||||
# downstream in the graph to finish its tasks before it passes through another
|
||||
# image. All images that come in while waiting are dropped, limiting the number
|
||||
# of in-flight images in HandDetectionSubgraph to 1. This prevents the nodes in
|
||||
# HandDetectionSubgraph from queuing up incoming images and data excessively,
|
||||
# which leads to increased latency and memory usage, unwanted in real-time
|
||||
# mobile applications. It also eliminates unnecessarily computation, e.g., the
|
||||
# output produced by a node in the subgraph may get dropped downstream if the
|
||||
# of in-flight images in PalmDetectionGpu to 1. This prevents the nodes in
|
||||
# PalmDetectionGpu from queuing up incoming images and data excessively, which
|
||||
# leads to increased latency and memory usage, unwanted in real-time mobile
|
||||
# applications. It also eliminates unnecessarily computation, e.g., the output
|
||||
# produced by a node in the subgraph may get dropped downstream if the
|
||||
# subsequent nodes are still busy processing previous inputs.
|
||||
node {
|
||||
calculator: "FlowLimiterCalculator"
|
||||
input_stream: "input_video"
|
||||
input_stream: "FINISHED:hand_rect_from_palm_detections"
|
||||
input_stream: "FINISHED:output_video"
|
||||
input_stream_info: {
|
||||
tag_index: "FINISHED"
|
||||
back_edge: true
|
||||
@@ -28,12 +30,11 @@ node {
|
||||
output_stream: "throttled_input_video"
|
||||
}
|
||||
|
||||
# Subgraph that detections hands (see hand_detection_gpu.pbtxt).
|
||||
# Detects palms.
|
||||
node {
|
||||
calculator: "HandDetectionSubgraph"
|
||||
input_stream: "throttled_input_video"
|
||||
calculator: "PalmDetectionGpu"
|
||||
input_stream: "IMAGE:throttled_input_video"
|
||||
output_stream: "DETECTIONS:palm_detections"
|
||||
output_stream: "NORM_RECT:hand_rect_from_palm_detections"
|
||||
}
|
||||
|
||||
# Converts detections to drawing primitives for annotation overlay.
|
||||
@@ -49,25 +50,10 @@ node {
|
||||
}
|
||||
}
|
||||
|
||||
# Converts normalized rects to drawing primitives for annotation overlay.
|
||||
node {
|
||||
calculator: "RectToRenderDataCalculator"
|
||||
input_stream: "NORM_RECT:hand_rect_from_palm_detections"
|
||||
output_stream: "RENDER_DATA:rect_render_data"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.RectToRenderDataCalculatorOptions] {
|
||||
filled: false
|
||||
color { r: 255 g: 0 b: 0 }
|
||||
thickness: 4.0
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Draws annotations and overlays them on top of the input images.
|
||||
node {
|
||||
calculator: "AnnotationOverlayCalculator"
|
||||
input_stream: "IMAGE_GPU:throttled_input_video"
|
||||
input_stream: "detection_render_data"
|
||||
input_stream: "rect_render_data"
|
||||
output_stream: "IMAGE_GPU:output_video"
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# MediaPipe graph that performs hand tracking on desktop with TensorFlow Lite
|
||||
# MediaPipe graph that performs hands tracking on desktop with TensorFlow Lite
|
||||
# on CPU.
|
||||
# Used in the example in
|
||||
# mediapipe/examples/desktop/hand_tracking:hand_tracking_tflite.
|
||||
@@ -16,99 +16,39 @@ node {
|
||||
output_stream: "VIDEO_PRESTREAM:input_video_header"
|
||||
}
|
||||
|
||||
# Caches a hand-presence decision fed back from HandLandmarkSubgraph, and upon
|
||||
# the arrival of the next input image sends out the cached decision with the
|
||||
# timestamp replaced by that of the input image, essentially generating a packet
|
||||
# that carries the previous hand-presence decision. Note that upon the arrival
|
||||
# of the very first input image, an empty packet is sent out to jump start the
|
||||
# feedback loop.
|
||||
# Generates side packet cotaining max number of hands to detect/track.
|
||||
node {
|
||||
calculator: "PreviousLoopbackCalculator"
|
||||
input_stream: "MAIN:input_video"
|
||||
input_stream: "LOOP:hand_presence"
|
||||
input_stream_info: {
|
||||
tag_index: "LOOP"
|
||||
back_edge: true
|
||||
}
|
||||
output_stream: "PREV_LOOP:prev_hand_presence"
|
||||
}
|
||||
|
||||
# Drops the incoming image if HandLandmarkSubgraph was able to identify hand
|
||||
# presence in the previous image. Otherwise, passes the incoming image through
|
||||
# to trigger a new round of hand detection in HandDetectionSubgraph.
|
||||
node {
|
||||
calculator: "GateCalculator"
|
||||
input_stream: "input_video"
|
||||
input_stream: "DISALLOW:prev_hand_presence"
|
||||
output_stream: "hand_detection_input_video"
|
||||
|
||||
calculator: "ConstantSidePacketCalculator"
|
||||
output_side_packet: "PACKET:num_hands"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.GateCalculatorOptions] {
|
||||
empty_packets_as_allow: true
|
||||
[type.googleapis.com/mediapipe.ConstantSidePacketCalculatorOptions]: {
|
||||
packet { int_value: 2 }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Subgraph that detections hands (see hand_detection_cpu.pbtxt).
|
||||
# Detects/tracks hand landmarks.
|
||||
node {
|
||||
calculator: "HandDetectionSubgraph"
|
||||
input_stream: "hand_detection_input_video"
|
||||
output_stream: "DETECTIONS:palm_detections"
|
||||
output_stream: "NORM_RECT:hand_rect_from_palm_detections"
|
||||
}
|
||||
|
||||
# Subgraph that localizes hand landmarks (see hand_landmark_cpu.pbtxt).
|
||||
node {
|
||||
calculator: "HandLandmarkSubgraph"
|
||||
calculator: "HandLandmarkTrackingCpu"
|
||||
input_stream: "IMAGE:input_video"
|
||||
input_stream: "NORM_RECT:hand_rect"
|
||||
output_stream: "LANDMARKS:hand_landmarks"
|
||||
output_stream: "NORM_RECT:hand_rect_from_landmarks"
|
||||
input_side_packet: "NUM_HANDS:num_hands"
|
||||
output_stream: "LANDMARKS:landmarks"
|
||||
output_stream: "HANDEDNESS:handedness"
|
||||
output_stream: "PRESENCE:hand_presence"
|
||||
}
|
||||
|
||||
# Caches a hand rectangle fed back from HandLandmarkSubgraph, and upon the
|
||||
# arrival of the next input image sends out the cached rectangle with the
|
||||
# timestamp replaced by that of the input image, essentially generating a packet
|
||||
# that carries the previous hand rectangle. Note that upon the arrival of the
|
||||
# very first input image, an empty packet is sent out to jump start the
|
||||
# feedback loop.
|
||||
node {
|
||||
calculator: "PreviousLoopbackCalculator"
|
||||
input_stream: "MAIN:input_video"
|
||||
input_stream: "LOOP:hand_rect_from_landmarks"
|
||||
input_stream_info: {
|
||||
tag_index: "LOOP"
|
||||
back_edge: true
|
||||
}
|
||||
output_stream: "PREV_LOOP:prev_hand_rect_from_landmarks"
|
||||
}
|
||||
|
||||
# Merges a stream of hand rectangles generated by HandDetectionSubgraph and that
|
||||
# generated by HandLandmarkSubgraph into a single output stream by selecting
|
||||
# between one of the two streams. The former is selected if the incoming packet
|
||||
# is not empty, i.e., hand detection is performed on the current image by
|
||||
# HandDetectionSubgraph (because HandLandmarkSubgraph could not identify hand
|
||||
# presence in the previous image). Otherwise, the latter is selected, which is
|
||||
# never empty because HandLandmarkSubgraphs processes all images (that went
|
||||
# through FlowLimiterCaculator).
|
||||
node {
|
||||
calculator: "MergeCalculator"
|
||||
input_stream: "hand_rect_from_palm_detections"
|
||||
input_stream: "prev_hand_rect_from_landmarks"
|
||||
output_stream: "hand_rect"
|
||||
output_stream: "PALM_DETECTIONS:multi_palm_detections"
|
||||
output_stream: "HAND_ROIS_FROM_LANDMARKS:multi_hand_rects"
|
||||
output_stream: "HAND_ROIS_FROM_PALM_DETECTIONS:multi_palm_rects"
|
||||
}
|
||||
|
||||
# Subgraph that renders annotations and overlays them on top of the input
|
||||
# images (see renderer_cpu.pbtxt).
|
||||
# images (see hand_renderer_cpu.pbtxt).
|
||||
node {
|
||||
calculator: "RendererSubgraph"
|
||||
calculator: "HandRendererSubgraph"
|
||||
input_stream: "IMAGE:input_video"
|
||||
input_stream: "LANDMARKS:hand_landmarks"
|
||||
input_stream: "NORM_RECT:hand_rect"
|
||||
input_stream: "DETECTIONS:palm_detections"
|
||||
input_stream: "DETECTIONS:multi_palm_detections"
|
||||
input_stream: "LANDMARKS:landmarks"
|
||||
input_stream: "HANDEDNESS:handedness"
|
||||
input_stream: "NORM_RECTS:0:multi_palm_rects"
|
||||
input_stream: "NORM_RECTS:1:multi_hand_rects"
|
||||
output_stream: "IMAGE:output_video"
|
||||
}
|
||||
|
||||
|
||||
@@ -1,108 +1,46 @@
|
||||
# MediaPipe graph that performs hand tracking on desktop with TensorFlow Lite
|
||||
# on CPU.
|
||||
# MediaPipe graph that performs hands tracking on desktop with TensorFlow
|
||||
# Lite on CPU.
|
||||
# Used in the example in
|
||||
# mediapipie/examples/desktop/hand_tracking:hand_tracking_cpu.
|
||||
# mediapipe/examples/desktop/hand_tracking:hand_tracking_cpu.
|
||||
|
||||
# Images coming into and out of the graph.
|
||||
# CPU image. (ImageFrame)
|
||||
input_stream: "input_video"
|
||||
|
||||
# CPU image. (ImageFrame)
|
||||
output_stream: "output_video"
|
||||
# Hand landmarks and palm detection info.
|
||||
output_stream: "palm_detections"
|
||||
output_stream: "hand_landmarks"
|
||||
|
||||
# Caches a hand-presence decision fed back from HandLandmarkSubgraph, and upon
|
||||
# the arrival of the next input image sends out the cached decision with the
|
||||
# timestamp replaced by that of the input image, essentially generating a packet
|
||||
# that carries the previous hand-presence decision. Note that upon the arrival
|
||||
# of the very first input image, an empty packet is sent out to jump start the
|
||||
# feedback loop.
|
||||
# Generates side packet cotaining max number of hands to detect/track.
|
||||
node {
|
||||
calculator: "PreviousLoopbackCalculator"
|
||||
input_stream: "MAIN:input_video"
|
||||
input_stream: "LOOP:hand_presence"
|
||||
input_stream_info: {
|
||||
tag_index: "LOOP"
|
||||
back_edge: true
|
||||
}
|
||||
output_stream: "PREV_LOOP:prev_hand_presence"
|
||||
}
|
||||
|
||||
# Drops the incoming image if HandLandmarkSubgraph was able to identify hand
|
||||
# presence in the previous image. Otherwise, passes the incoming image through
|
||||
# to trigger a new round of hand detection in HandDetectionSubgraph.
|
||||
node {
|
||||
calculator: "GateCalculator"
|
||||
input_stream: "input_video"
|
||||
input_stream: "DISALLOW:prev_hand_presence"
|
||||
output_stream: "hand_detection_input_video"
|
||||
|
||||
calculator: "ConstantSidePacketCalculator"
|
||||
output_side_packet: "PACKET:num_hands"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.GateCalculatorOptions] {
|
||||
empty_packets_as_allow: true
|
||||
[type.googleapis.com/mediapipe.ConstantSidePacketCalculatorOptions]: {
|
||||
packet { int_value: 2 }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Subgraph that detections hands (see hand_detection_cpu.pbtxt).
|
||||
# Detects/tracks hand landmarks.
|
||||
node {
|
||||
calculator: "HandDetectionSubgraph"
|
||||
input_stream: "hand_detection_input_video"
|
||||
output_stream: "DETECTIONS:palm_detections"
|
||||
output_stream: "NORM_RECT:hand_rect_from_palm_detections"
|
||||
}
|
||||
|
||||
# Subgraph that localizes hand landmarks (see hand_landmark_cpu.pbtxt).
|
||||
node {
|
||||
calculator: "HandLandmarkSubgraph"
|
||||
calculator: "HandLandmarkTrackingCpu"
|
||||
input_stream: "IMAGE:input_video"
|
||||
input_stream: "NORM_RECT:hand_rect"
|
||||
output_stream: "LANDMARKS:hand_landmarks"
|
||||
output_stream: "NORM_RECT:hand_rect_from_landmarks"
|
||||
input_side_packet: "NUM_HANDS:num_hands"
|
||||
output_stream: "LANDMARKS:landmarks"
|
||||
output_stream: "HANDEDNESS:handedness"
|
||||
output_stream: "PRESENCE:hand_presence"
|
||||
}
|
||||
|
||||
# Caches a hand rectangle fed back from HandLandmarkSubgraph, and upon the
|
||||
# arrival of the next input image sends out the cached rectangle with the
|
||||
# timestamp replaced by that of the input image, essentially generating a packet
|
||||
# that carries the previous hand rectangle. Note that upon the arrival of the
|
||||
# very first input image, an empty packet is sent out to jump start the
|
||||
# feedback loop.
|
||||
node {
|
||||
calculator: "PreviousLoopbackCalculator"
|
||||
input_stream: "MAIN:input_video"
|
||||
input_stream: "LOOP:hand_rect_from_landmarks"
|
||||
input_stream_info: {
|
||||
tag_index: "LOOP"
|
||||
back_edge: true
|
||||
}
|
||||
output_stream: "PREV_LOOP:prev_hand_rect_from_landmarks"
|
||||
}
|
||||
|
||||
# Merges a stream of hand rectangles generated by HandDetectionSubgraph and that
|
||||
# generated by HandLandmarkSubgraph into a single output stream by selecting
|
||||
# between one of the two streams. The former is selected if the incoming packet
|
||||
# is not empty, i.e., hand detection is performed on the current image by
|
||||
# HandDetectionSubgraph (because HandLandmarkSubgraph could not identify hand
|
||||
# presence in the previous image). Otherwise, the latter is selected, which is
|
||||
# never empty because HandLandmarkSubgraphs processes all images (that went
|
||||
# through FlowLimiterCaculator).
|
||||
node {
|
||||
calculator: "MergeCalculator"
|
||||
input_stream: "hand_rect_from_palm_detections"
|
||||
input_stream: "prev_hand_rect_from_landmarks"
|
||||
output_stream: "hand_rect"
|
||||
output_stream: "PALM_DETECTIONS:multi_palm_detections"
|
||||
output_stream: "HAND_ROIS_FROM_LANDMARKS:multi_hand_rects"
|
||||
output_stream: "HAND_ROIS_FROM_PALM_DETECTIONS:multi_palm_rects"
|
||||
}
|
||||
|
||||
# Subgraph that renders annotations and overlays them on top of the input
|
||||
# images (see renderer_cpu.pbtxt).
|
||||
# images (see hand_renderer_cpu.pbtxt).
|
||||
node {
|
||||
calculator: "RendererSubgraph"
|
||||
calculator: "HandRendererSubgraph"
|
||||
input_stream: "IMAGE:input_video"
|
||||
input_stream: "LANDMARKS:hand_landmarks"
|
||||
input_stream: "NORM_RECT:hand_rect"
|
||||
input_stream: "DETECTIONS:palm_detections"
|
||||
input_stream: "DETECTIONS:multi_palm_detections"
|
||||
input_stream: "LANDMARKS:landmarks"
|
||||
input_stream: "HANDEDNESS:handedness"
|
||||
input_stream: "NORM_RECTS:0:multi_palm_rects"
|
||||
input_stream: "NORM_RECTS:1:multi_hand_rects"
|
||||
output_stream: "IMAGE:output_video"
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
# MediaPipe graph that performs multi-hand tracking with TensorFlow Lite on GPU.
|
||||
# Used in the examples in
|
||||
# mediapipe/examples/android/src/java/com/mediapipe/apps/handtrackinggpu.
|
||||
|
||||
# GPU image. (GpuBuffer)
|
||||
input_stream: "input_video"
|
||||
|
||||
# GPU image. (GpuBuffer)
|
||||
output_stream: "output_video"
|
||||
# Collection of detected/predicted hands, each represented as a list of
|
||||
# landmarks. (std::vector<NormalizedLandmarkList>)
|
||||
output_stream: "hand_landmarks"
|
||||
|
||||
# Generates side packet cotaining max number of hands to detect/track.
|
||||
node {
|
||||
calculator: "ConstantSidePacketCalculator"
|
||||
output_side_packet: "PACKET:num_hands"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.ConstantSidePacketCalculatorOptions]: {
|
||||
packet { int_value: 2 }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Detects/tracks hand landmarks.
|
||||
node {
|
||||
calculator: "HandLandmarkTrackingGpu"
|
||||
input_stream: "IMAGE:input_video"
|
||||
input_side_packet: "NUM_HANDS:num_hands"
|
||||
output_stream: "LANDMARKS:hand_landmarks"
|
||||
output_stream: "HANDEDNESS:handedness"
|
||||
output_stream: "PALM_DETECTIONS:palm_detections"
|
||||
output_stream: "HAND_ROIS_FROM_LANDMARKS:hand_rects_from_landmarks"
|
||||
output_stream: "HAND_ROIS_FROM_PALM_DETECTIONS:hand_rects_from_palm_detections"
|
||||
}
|
||||
|
||||
# Subgraph that renders annotations and overlays them on top of the input
|
||||
# images (see hand_renderer_gpu.pbtxt).
|
||||
node {
|
||||
calculator: "HandRendererSubgraph"
|
||||
input_stream: "IMAGE:input_video"
|
||||
input_stream: "DETECTIONS:palm_detections"
|
||||
input_stream: "LANDMARKS:hand_landmarks"
|
||||
input_stream: "HANDEDNESS:handedness"
|
||||
input_stream: "NORM_RECTS:0:hand_rects_from_palm_detections"
|
||||
input_stream: "NORM_RECTS:1:hand_rects_from_landmarks"
|
||||
output_stream: "IMAGE:output_video"
|
||||
}
|
||||
@@ -1,11 +1,18 @@
|
||||
# MediaPipe graph that performs hand tracking with TensorFlow Lite on GPU.
|
||||
# MediaPipe graph that performs multi-hand tracking with TensorFlow Lite on GPU.
|
||||
# Used in the examples in
|
||||
# mediapipe/examples/android/src/java/com/mediapipe/apps/handtrackinggpu and
|
||||
# mediapipe/examples/ios/handtrackinggpu.
|
||||
# mediapipe/examples/android/src/java/com/mediapipe/apps/handtrackinggpu.
|
||||
|
||||
# Images coming into and out of the graph.
|
||||
# GPU image. (GpuBuffer)
|
||||
input_stream: "input_video"
|
||||
|
||||
# Max number of hands to detect/process. (int)
|
||||
input_side_packet: "num_hands"
|
||||
|
||||
# GPU image. (GpuBuffer)
|
||||
output_stream: "output_video"
|
||||
# Collection of detected/predicted hands, each represented as a list of
|
||||
# landmarks. (std::vector<NormalizedLandmarkList>)
|
||||
output_stream: "hand_landmarks"
|
||||
|
||||
# Throttles the images flowing downstream for flow control. It passes through
|
||||
# the very first incoming image unaltered, and waits for downstream nodes
|
||||
@@ -20,7 +27,7 @@ output_stream: "output_video"
|
||||
node {
|
||||
calculator: "FlowLimiterCalculator"
|
||||
input_stream: "input_video"
|
||||
input_stream: "FINISHED:hand_rect"
|
||||
input_stream: "FINISHED:output_video"
|
||||
input_stream_info: {
|
||||
tag_index: "FINISHED"
|
||||
back_edge: true
|
||||
@@ -28,98 +35,27 @@ node {
|
||||
output_stream: "throttled_input_video"
|
||||
}
|
||||
|
||||
# Caches a hand-presence decision fed back from HandLandmarkSubgraph, and upon
|
||||
# the arrival of the next input image sends out the cached decision with the
|
||||
# timestamp replaced by that of the input image, essentially generating a packet
|
||||
# that carries the previous hand-presence decision. Note that upon the arrival
|
||||
# of the very first input image, an empty packet is sent out to jump start the
|
||||
# feedback loop.
|
||||
# Detects/tracks hand landmarks.
|
||||
node {
|
||||
calculator: "PreviousLoopbackCalculator"
|
||||
input_stream: "MAIN:throttled_input_video"
|
||||
input_stream: "LOOP:hand_presence"
|
||||
input_stream_info: {
|
||||
tag_index: "LOOP"
|
||||
back_edge: true
|
||||
}
|
||||
output_stream: "PREV_LOOP:prev_hand_presence"
|
||||
}
|
||||
|
||||
# Drops the incoming image if HandLandmarkSubgraph was able to identify hand
|
||||
# presence in the previous image. Otherwise, passes the incoming image through
|
||||
# to trigger a new round of hand detection in HandDetectionSubgraph.
|
||||
node {
|
||||
calculator: "GateCalculator"
|
||||
input_stream: "throttled_input_video"
|
||||
input_stream: "DISALLOW:prev_hand_presence"
|
||||
output_stream: "hand_detection_input_video"
|
||||
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.GateCalculatorOptions] {
|
||||
empty_packets_as_allow: true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Subgraph that detections hands (see hand_detection_gpu.pbtxt).
|
||||
node {
|
||||
calculator: "HandDetectionSubgraph"
|
||||
input_stream: "hand_detection_input_video"
|
||||
output_stream: "DETECTIONS:palm_detections"
|
||||
output_stream: "NORM_RECT:hand_rect_from_palm_detections"
|
||||
}
|
||||
|
||||
# Subgraph that localizes hand landmarks (see hand_landmark_gpu.pbtxt).
|
||||
node {
|
||||
calculator: "HandLandmarkSubgraph"
|
||||
calculator: "HandLandmarkTrackingGpu"
|
||||
input_stream: "IMAGE:throttled_input_video"
|
||||
input_stream: "NORM_RECT:hand_rect"
|
||||
input_side_packet: "NUM_HANDS:num_hands"
|
||||
output_stream: "LANDMARKS:hand_landmarks"
|
||||
output_stream: "NORM_RECT:hand_rect_from_landmarks"
|
||||
output_stream: "PRESENCE:hand_presence"
|
||||
output_stream: "HANDEDNESS:handedness"
|
||||
}
|
||||
|
||||
# Caches a hand rectangle fed back from HandLandmarkSubgraph, and upon the
|
||||
# arrival of the next input image sends out the cached rectangle with the
|
||||
# timestamp replaced by that of the input image, essentially generating a packet
|
||||
# that carries the previous hand rectangle. Note that upon the arrival of the
|
||||
# very first input image, an empty packet is sent out to jump start the
|
||||
# feedback loop.
|
||||
node {
|
||||
calculator: "PreviousLoopbackCalculator"
|
||||
input_stream: "MAIN:throttled_input_video"
|
||||
input_stream: "LOOP:hand_rect_from_landmarks"
|
||||
input_stream_info: {
|
||||
tag_index: "LOOP"
|
||||
back_edge: true
|
||||
}
|
||||
output_stream: "PREV_LOOP:prev_hand_rect_from_landmarks"
|
||||
}
|
||||
|
||||
# Merges a stream of hand rectangles generated by HandDetectionSubgraph and that
|
||||
# generated by HandLandmarkSubgraph into a single output stream by selecting
|
||||
# between one of the two streams. The former is selected if the incoming packet
|
||||
# is not empty, i.e., hand detection is performed on the current image by
|
||||
# HandDetectionSubgraph (because HandLandmarkSubgraph could not identify hand
|
||||
# presence in the previous image). Otherwise, the latter is selected, which is
|
||||
# never empty because HandLandmarkSubgraphs processes all images (that went
|
||||
# through FlowLimiterCaculator).
|
||||
node {
|
||||
calculator: "MergeCalculator"
|
||||
input_stream: "hand_rect_from_palm_detections"
|
||||
input_stream: "prev_hand_rect_from_landmarks"
|
||||
output_stream: "hand_rect"
|
||||
output_stream: "PALM_DETECTIONS:palm_detections"
|
||||
output_stream: "HAND_ROIS_FROM_LANDMARKS:hand_rects_from_landmarks"
|
||||
output_stream: "HAND_ROIS_FROM_PALM_DETECTIONS:hand_rects_from_palm_detections"
|
||||
}
|
||||
|
||||
# Subgraph that renders annotations and overlays them on top of the input
|
||||
# images (see renderer_gpu.pbtxt).
|
||||
# images (see hand_renderer_gpu.pbtxt).
|
||||
node {
|
||||
calculator: "RendererSubgraph"
|
||||
calculator: "HandRendererSubgraph"
|
||||
input_stream: "IMAGE:throttled_input_video"
|
||||
input_stream: "LANDMARKS:hand_landmarks"
|
||||
input_stream: "NORM_RECT:hand_rect"
|
||||
input_stream: "DETECTIONS:palm_detections"
|
||||
input_stream: "LANDMARKS:hand_landmarks"
|
||||
input_stream: "HANDEDNESS:handedness"
|
||||
input_stream: "NORM_RECTS:0:hand_rects_from_palm_detections"
|
||||
input_stream: "NORM_RECTS:1:hand_rects_from_landmarks"
|
||||
output_stream: "IMAGE:output_video"
|
||||
}
|
||||
|
||||
@@ -1,127 +0,0 @@
|
||||
# MediaPipe graph that performs multi-hand tracking on desktop with TensorFlow
|
||||
# Lite on CPU.
|
||||
# Used in the example in
|
||||
# mediapipie/examples/desktop/hand_tracking:multi_hand_tracking_tflite.
|
||||
|
||||
# max_queue_size limits the number of packets enqueued on any input stream
|
||||
# by throttling inputs to the graph. This makes the graph only process one
|
||||
# frame per time.
|
||||
max_queue_size: 1
|
||||
|
||||
# Decodes an input video file into images and a video header.
|
||||
node {
|
||||
calculator: "OpenCvVideoDecoderCalculator"
|
||||
input_side_packet: "INPUT_FILE_PATH:input_video_path"
|
||||
output_stream: "VIDEO:input_video"
|
||||
output_stream: "VIDEO_PRESTREAM:input_video_header"
|
||||
}
|
||||
|
||||
# Determines if an input vector of NormalizedRect has a size greater than or
|
||||
# equal to the provided min_size.
|
||||
node {
|
||||
calculator: "NormalizedRectVectorHasMinSizeCalculator"
|
||||
input_stream: "ITERABLE:prev_multi_hand_rects_from_landmarks"
|
||||
output_stream: "prev_has_enough_hands"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.CollectionHasMinSizeCalculatorOptions] {
|
||||
# This value can be changed to support tracking arbitrary number of hands.
|
||||
# Please also remember to modify max_vec_size in
|
||||
# ClipVectorSizeCalculatorOptions in
|
||||
# mediapipe/graphs/hand_tracking/subgraphs/multi_hand_detection_cpu.pbtxt
|
||||
min_size: 2
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Drops the incoming image if the previous frame had at least N hands.
|
||||
# Otherwise, passes the incoming image through to trigger a new round of hand
|
||||
# detection in MultiHandDetectionSubgraph.
|
||||
node {
|
||||
calculator: "GateCalculator"
|
||||
input_stream: "input_video"
|
||||
input_stream: "DISALLOW:prev_has_enough_hands"
|
||||
output_stream: "multi_hand_detection_input_video"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.GateCalculatorOptions] {
|
||||
empty_packets_as_allow: true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Subgraph that detections hands (see multi_hand_detection_cpu.pbtxt).
|
||||
node {
|
||||
calculator: "MultiHandDetectionSubgraph"
|
||||
input_stream: "multi_hand_detection_input_video"
|
||||
output_stream: "DETECTIONS:multi_palm_detections"
|
||||
output_stream: "NORM_RECTS:multi_palm_rects"
|
||||
}
|
||||
|
||||
# Subgraph that localizes hand landmarks for multiple hands (see
|
||||
# multi_hand_landmark.pbtxt).
|
||||
node {
|
||||
calculator: "MultiHandLandmarkSubgraph"
|
||||
input_stream: "IMAGE:input_video"
|
||||
input_stream: "NORM_RECTS:multi_hand_rects"
|
||||
output_stream: "LANDMARKS:multi_hand_landmarks"
|
||||
output_stream: "NORM_RECTS:multi_hand_rects_from_landmarks"
|
||||
}
|
||||
|
||||
# Caches a hand rectangle fed back from MultiHandLandmarkSubgraph, and upon the
|
||||
# arrival of the next input image sends out the cached rectangle with the
|
||||
# timestamp replaced by that of the input image, essentially generating a packet
|
||||
# that carries the previous hand rectangle. Note that upon the arrival of the
|
||||
# very first input image, an empty packet is sent out to jump start the
|
||||
# feedback loop.
|
||||
node {
|
||||
calculator: "PreviousLoopbackCalculator"
|
||||
input_stream: "MAIN:input_video"
|
||||
input_stream: "LOOP:multi_hand_rects_from_landmarks"
|
||||
input_stream_info: {
|
||||
tag_index: "LOOP"
|
||||
back_edge: true
|
||||
}
|
||||
output_stream: "PREV_LOOP:prev_multi_hand_rects_from_landmarks"
|
||||
}
|
||||
|
||||
# Performs association between NormalizedRect vector elements from previous
|
||||
# frame and those from the current frame if MultiHandDetectionSubgraph runs.
|
||||
# This calculator ensures that the output multi_hand_rects vector doesn't
|
||||
# contain overlapping regions based on the specified min_similarity_threshold.
|
||||
node {
|
||||
calculator: "AssociationNormRectCalculator"
|
||||
input_stream: "prev_multi_hand_rects_from_landmarks"
|
||||
input_stream: "multi_palm_rects"
|
||||
output_stream: "multi_hand_rects"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.AssociationCalculatorOptions] {
|
||||
min_similarity_threshold: 0.5
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Subgraph that renders annotations and overlays them on top of the input
|
||||
# images (see multi_hand_renderer_cpu.pbtxt).
|
||||
node {
|
||||
calculator: "MultiHandRendererSubgraph"
|
||||
input_stream: "IMAGE:input_video"
|
||||
input_stream: "DETECTIONS:multi_palm_detections"
|
||||
input_stream: "LANDMARKS:multi_hand_landmarks"
|
||||
input_stream: "NORM_RECTS:0:multi_palm_rects"
|
||||
input_stream: "NORM_RECTS:1:multi_hand_rects"
|
||||
output_stream: "IMAGE:output_video"
|
||||
}
|
||||
|
||||
# Encodes the annotated images into a video file, adopting properties specified
|
||||
# in the input video header, e.g., video framerate.
|
||||
node {
|
||||
calculator: "OpenCvVideoEncoderCalculator"
|
||||
input_stream: "VIDEO:output_video"
|
||||
input_stream: "VIDEO_PRESTREAM:input_video_header"
|
||||
input_side_packet: "OUTPUT_FILE_PATH:output_video_path"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.OpenCvVideoEncoderCalculatorOptions]: {
|
||||
codec: "avc1"
|
||||
video_format: "mp4"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,106 +0,0 @@
|
||||
# MediaPipe graph that performs multi-hand tracking on desktop with TensorFlow
|
||||
# Lite on CPU.
|
||||
# Used in the example in
|
||||
# mediapipie/examples/desktop/hand_tracking:multi_hand_tracking_cpu.
|
||||
|
||||
# Images coming into and out of the graph.
|
||||
input_stream: "input_video"
|
||||
output_stream: "output_video"
|
||||
# Palm detections and hand landmarks info.
|
||||
output_stream: "multi_palm_detections"
|
||||
output_stream: "multi_hand_landmarks"
|
||||
|
||||
# Determines if an input vector of NormalizedRect has a size greater than or
|
||||
# equal to the provided min_size.
|
||||
node {
|
||||
calculator: "NormalizedRectVectorHasMinSizeCalculator"
|
||||
input_stream: "ITERABLE:prev_multi_hand_rects_from_landmarks"
|
||||
output_stream: "prev_has_enough_hands"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.CollectionHasMinSizeCalculatorOptions] {
|
||||
# This value can be changed to support tracking arbitrary number of hands.
|
||||
# Please also remember to modify max_vec_size in
|
||||
# ClipVectorSizeCalculatorOptions in
|
||||
# mediapipe/graphs/hand_tracking/subgraphs/multi_hand_detection_gpu.pbtxt
|
||||
min_size: 2
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Drops the incoming image if the previous frame had at least N hands.
|
||||
# Otherwise, passes the incoming image through to trigger a new round of hand
|
||||
# detection in MultiHandDetectionSubgraph.
|
||||
node {
|
||||
calculator: "GateCalculator"
|
||||
input_stream: "input_video"
|
||||
input_stream: "DISALLOW:prev_has_enough_hands"
|
||||
output_stream: "multi_hand_detection_input_video"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.GateCalculatorOptions] {
|
||||
empty_packets_as_allow: true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Subgraph that detections hands (see multi_hand_detection_cpu.pbtxt).
|
||||
node {
|
||||
calculator: "MultiHandDetectionSubgraph"
|
||||
input_stream: "multi_hand_detection_input_video"
|
||||
output_stream: "DETECTIONS:multi_palm_detections"
|
||||
output_stream: "NORM_RECTS:multi_palm_rects"
|
||||
}
|
||||
|
||||
# Subgraph that localizes hand landmarks for multiple hands (see
|
||||
# multi_hand_landmark.pbtxt).
|
||||
node {
|
||||
calculator: "MultiHandLandmarkSubgraph"
|
||||
input_stream: "IMAGE:input_video"
|
||||
input_stream: "NORM_RECTS:multi_hand_rects"
|
||||
output_stream: "LANDMARKS:multi_hand_landmarks"
|
||||
output_stream: "NORM_RECTS:multi_hand_rects_from_landmarks"
|
||||
}
|
||||
|
||||
# Caches a hand rectangle fed back from MultiHandLandmarkSubgraph, and upon the
|
||||
# arrival of the next input image sends out the cached rectangle with the
|
||||
# timestamp replaced by that of the input image, essentially generating a packet
|
||||
# that carries the previous hand rectangle. Note that upon the arrival of the
|
||||
# very first input image, an empty packet is sent out to jump start the
|
||||
# feedback loop.
|
||||
node {
|
||||
calculator: "PreviousLoopbackCalculator"
|
||||
input_stream: "MAIN:input_video"
|
||||
input_stream: "LOOP:multi_hand_rects_from_landmarks"
|
||||
input_stream_info: {
|
||||
tag_index: "LOOP"
|
||||
back_edge: true
|
||||
}
|
||||
output_stream: "PREV_LOOP:prev_multi_hand_rects_from_landmarks"
|
||||
}
|
||||
|
||||
# Performs association between NormalizedRect vector elements from previous
|
||||
# frame and those from the current frame if MultiHandDetectionSubgraph runs.
|
||||
# This calculator ensures that the output multi_hand_rects vector doesn't
|
||||
# contain overlapping regions based on the specified min_similarity_threshold.
|
||||
node {
|
||||
calculator: "AssociationNormRectCalculator"
|
||||
input_stream: "prev_multi_hand_rects_from_landmarks"
|
||||
input_stream: "multi_palm_rects"
|
||||
output_stream: "multi_hand_rects"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.AssociationCalculatorOptions] {
|
||||
min_similarity_threshold: 0.5
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Subgraph that renders annotations and overlays them on top of the input
|
||||
# images (see multi_hand_renderer_cpu.pbtxt).
|
||||
node {
|
||||
calculator: "MultiHandRendererSubgraph"
|
||||
input_stream: "IMAGE:input_video"
|
||||
input_stream: "DETECTIONS:multi_palm_detections"
|
||||
input_stream: "LANDMARKS:multi_hand_landmarks"
|
||||
input_stream: "NORM_RECTS:0:multi_palm_rects"
|
||||
input_stream: "NORM_RECTS:1:multi_hand_rects"
|
||||
output_stream: "IMAGE:output_video"
|
||||
}
|
||||
@@ -1,123 +0,0 @@
|
||||
# MediaPipe graph that performs multi-hand tracking with TensorFlow Lite on GPU.
|
||||
# Used in the examples in
|
||||
# mediapipe/examples/android/src/java/com/mediapipe/apps/multihandtrackinggpu.
|
||||
|
||||
# Images coming into and out of the graph.
|
||||
input_stream: "input_video"
|
||||
output_stream: "output_video"
|
||||
|
||||
# Throttles the images flowing downstream for flow control. It passes through
|
||||
# the very first incoming image unaltered, and waits for downstream nodes
|
||||
# (calculators and subgraphs) in the graph to finish their tasks before it
|
||||
# passes through another image. All images that come in while waiting are
|
||||
# dropped, limiting the number of in-flight images in most part of the graph to
|
||||
# 1. This prevents the downstream nodes from queuing up incoming images and data
|
||||
# excessively, which leads to increased latency and memory usage, unwanted in
|
||||
# real-time mobile applications. It also eliminates unnecessarily computation,
|
||||
# e.g., the output produced by a node may get dropped downstream if the
|
||||
# subsequent nodes are still busy processing previous inputs.
|
||||
node {
|
||||
calculator: "FlowLimiterCalculator"
|
||||
input_stream: "input_video"
|
||||
input_stream: "FINISHED:multi_hand_rects"
|
||||
input_stream_info: {
|
||||
tag_index: "FINISHED"
|
||||
back_edge: true
|
||||
}
|
||||
output_stream: "throttled_input_video"
|
||||
}
|
||||
|
||||
# Determines if an input vector of NormalizedRect has a size greater than or
|
||||
# equal to the provided min_size.
|
||||
node {
|
||||
calculator: "NormalizedRectVectorHasMinSizeCalculator"
|
||||
input_stream: "ITERABLE:prev_multi_hand_rects_from_landmarks"
|
||||
output_stream: "prev_has_enough_hands"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.CollectionHasMinSizeCalculatorOptions] {
|
||||
# This value can be changed to support tracking arbitrary number of hands.
|
||||
# Please also remember to modify max_vec_size in
|
||||
# ClipVectorSizeCalculatorOptions in
|
||||
# mediapipe/graphs/hand_tracking/subgraphs/multi_hand_detection_gpu.pbtxt
|
||||
min_size: 2
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Drops the incoming image if the previous frame had at least N hands.
|
||||
# Otherwise, passes the incoming image through to trigger a new round of hand
|
||||
# detection in MultiHandDetectionSubgraph.
|
||||
node {
|
||||
calculator: "GateCalculator"
|
||||
input_stream: "throttled_input_video"
|
||||
input_stream: "DISALLOW:prev_has_enough_hands"
|
||||
output_stream: "multi_hand_detection_input_video"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.GateCalculatorOptions] {
|
||||
empty_packets_as_allow: true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Subgraph that detections hands (see multi_hand_detection_gpu.pbtxt).
|
||||
node {
|
||||
calculator: "MultiHandDetectionSubgraph"
|
||||
input_stream: "multi_hand_detection_input_video"
|
||||
output_stream: "DETECTIONS:multi_palm_detections"
|
||||
output_stream: "NORM_RECTS:multi_palm_rects"
|
||||
}
|
||||
|
||||
# Subgraph that localizes hand landmarks for multiple hands (see
|
||||
# multi_hand_landmark.pbtxt).
|
||||
node {
|
||||
calculator: "MultiHandLandmarkSubgraph"
|
||||
input_stream: "IMAGE:throttled_input_video"
|
||||
input_stream: "NORM_RECTS:multi_hand_rects"
|
||||
output_stream: "LANDMARKS:multi_hand_landmarks"
|
||||
output_stream: "NORM_RECTS:multi_hand_rects_from_landmarks"
|
||||
}
|
||||
|
||||
# Caches a hand rectangle fed back from MultiHandLandmarkSubgraph, and upon the
|
||||
# arrival of the next input image sends out the cached rectangle with the
|
||||
# timestamp replaced by that of the input image, essentially generating a packet
|
||||
# that carries the previous hand rectangle. Note that upon the arrival of the
|
||||
# very first input image, an empty packet is sent out to jump start the
|
||||
# feedback loop.
|
||||
node {
|
||||
calculator: "PreviousLoopbackCalculator"
|
||||
input_stream: "MAIN:throttled_input_video"
|
||||
input_stream: "LOOP:multi_hand_rects_from_landmarks"
|
||||
input_stream_info: {
|
||||
tag_index: "LOOP"
|
||||
back_edge: true
|
||||
}
|
||||
output_stream: "PREV_LOOP:prev_multi_hand_rects_from_landmarks"
|
||||
}
|
||||
|
||||
# Performs association between NormalizedRect vector elements from previous
|
||||
# frame and those from the current frame if MultiHandDetectionSubgraph runs.
|
||||
# This calculator ensures that the output multi_hand_rects vector doesn't
|
||||
# contain overlapping regions based on the specified min_similarity_threshold.
|
||||
node {
|
||||
calculator: "AssociationNormRectCalculator"
|
||||
input_stream: "prev_multi_hand_rects_from_landmarks"
|
||||
input_stream: "multi_palm_rects"
|
||||
output_stream: "multi_hand_rects"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.AssociationCalculatorOptions] {
|
||||
min_similarity_threshold: 0.5
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Subgraph that renders annotations and overlays them on top of the input
|
||||
# images (see multi_hand_renderer_gpu.pbtxt).
|
||||
node {
|
||||
calculator: "MultiHandRendererSubgraph"
|
||||
input_stream: "IMAGE:throttled_input_video"
|
||||
input_stream: "DETECTIONS:multi_palm_detections"
|
||||
input_stream: "LANDMARKS:multi_hand_landmarks"
|
||||
input_stream: "NORM_RECTS:0:multi_palm_rects"
|
||||
input_stream: "NORM_RECTS:1:multi_hand_rects"
|
||||
output_stream: "IMAGE:output_video"
|
||||
}
|
||||
@@ -22,94 +22,16 @@ licenses(["notice"])
|
||||
package(default_visibility = ["//visibility:public"])
|
||||
|
||||
mediapipe_simple_subgraph(
|
||||
name = "hand_detection_cpu",
|
||||
graph = "hand_detection_cpu.pbtxt",
|
||||
register_as = "HandDetectionSubgraph",
|
||||
deps = [
|
||||
"//mediapipe/calculators/image:image_properties_calculator",
|
||||
"//mediapipe/calculators/image:image_transformation_calculator",
|
||||
"//mediapipe/calculators/tflite:ssd_anchors_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_converter_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_custom_op_resolver_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_inference_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_tensors_to_detections_calculator",
|
||||
"//mediapipe/calculators/util:detection_label_id_to_text_calculator",
|
||||
"//mediapipe/calculators/util:detection_letterbox_removal_calculator",
|
||||
"//mediapipe/calculators/util:detections_to_rects_calculator",
|
||||
"//mediapipe/calculators/util:detections_to_render_data_calculator",
|
||||
"//mediapipe/calculators/util:non_max_suppression_calculator",
|
||||
"//mediapipe/calculators/util:rect_transformation_calculator",
|
||||
],
|
||||
)
|
||||
|
||||
mediapipe_simple_subgraph(
|
||||
name = "multi_hand_detection_cpu",
|
||||
graph = "multi_hand_detection_cpu.pbtxt",
|
||||
register_as = "MultiHandDetectionSubgraph",
|
||||
name = "hand_renderer_cpu",
|
||||
graph = "hand_renderer_cpu.pbtxt",
|
||||
register_as = "HandRendererSubgraph",
|
||||
deps = [
|
||||
"//mediapipe/calculators/core:begin_loop_calculator",
|
||||
"//mediapipe/calculators/core:clip_vector_size_calculator",
|
||||
"//mediapipe/calculators/core:end_loop_calculator",
|
||||
"//mediapipe/calculators/image:image_properties_calculator",
|
||||
"//mediapipe/calculators/image:image_transformation_calculator",
|
||||
"//mediapipe/calculators/tflite:ssd_anchors_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_converter_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_custom_op_resolver_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_inference_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_tensors_to_detections_calculator",
|
||||
"//mediapipe/calculators/util:detection_label_id_to_text_calculator",
|
||||
"//mediapipe/calculators/util:detection_letterbox_removal_calculator",
|
||||
"//mediapipe/calculators/util:detections_to_rects_calculator",
|
||||
"//mediapipe/calculators/util:non_max_suppression_calculator",
|
||||
"//mediapipe/calculators/util:rect_transformation_calculator",
|
||||
],
|
||||
)
|
||||
|
||||
mediapipe_simple_subgraph(
|
||||
name = "hand_landmark_cpu",
|
||||
graph = "hand_landmark_cpu.pbtxt",
|
||||
register_as = "HandLandmarkSubgraph",
|
||||
deps = [
|
||||
"//mediapipe/calculators/core:split_normalized_landmark_list_calculator",
|
||||
"//mediapipe/calculators/core:gate_calculator",
|
||||
"//mediapipe/calculators/core:split_vector_calculator",
|
||||
"//mediapipe/calculators/image:image_cropping_calculator",
|
||||
"//mediapipe/calculators/image:image_properties_calculator",
|
||||
"//mediapipe/calculators/image:image_transformation_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_converter_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_custom_op_resolver_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_inference_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_tensors_to_classification_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_tensors_to_floats_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_tensors_to_landmarks_calculator",
|
||||
"//mediapipe/calculators/util:detections_to_rects_calculator",
|
||||
"//mediapipe/calculators/util:landmark_letterbox_removal_calculator",
|
||||
"//mediapipe/calculators/util:landmark_projection_calculator",
|
||||
"//mediapipe/calculators/util:landmarks_to_detection_calculator",
|
||||
"//mediapipe/calculators/util:landmarks_to_render_data_calculator",
|
||||
"//mediapipe/calculators/util:rect_transformation_calculator",
|
||||
"//mediapipe/calculators/util:thresholding_calculator",
|
||||
"//mediapipe/graphs/hand_tracking/calculators:hand_landmarks_to_rect_calculator",
|
||||
],
|
||||
)
|
||||
|
||||
mediapipe_simple_subgraph(
|
||||
name = "multi_hand_landmark_cpu",
|
||||
graph = "multi_hand_landmark.pbtxt",
|
||||
register_as = "MultiHandLandmarkSubgraph",
|
||||
deps = [
|
||||
":hand_landmark_cpu",
|
||||
"//mediapipe/calculators/core:begin_loop_calculator",
|
||||
"//mediapipe/calculators/core:end_loop_calculator",
|
||||
"//mediapipe/calculators/util:filter_collection_calculator",
|
||||
],
|
||||
)
|
||||
|
||||
mediapipe_simple_subgraph(
|
||||
name = "renderer_cpu",
|
||||
graph = "renderer_cpu.pbtxt",
|
||||
register_as = "RendererSubgraph",
|
||||
deps = [
|
||||
"//mediapipe/calculators/util:annotation_overlay_calculator",
|
||||
"//mediapipe/calculators/util:collection_has_min_size_calculator",
|
||||
"//mediapipe/calculators/util:detections_to_render_data_calculator",
|
||||
"//mediapipe/calculators/util:labels_to_render_data_calculator",
|
||||
"//mediapipe/calculators/util:landmarks_to_render_data_calculator",
|
||||
@@ -118,123 +40,19 @@ mediapipe_simple_subgraph(
|
||||
)
|
||||
|
||||
mediapipe_simple_subgraph(
|
||||
name = "multi_hand_renderer_cpu",
|
||||
graph = "multi_hand_renderer_cpu.pbtxt",
|
||||
register_as = "MultiHandRendererSubgraph",
|
||||
name = "hand_renderer_gpu",
|
||||
graph = "hand_renderer_gpu.pbtxt",
|
||||
register_as = "HandRendererSubgraph",
|
||||
deps = [
|
||||
"//mediapipe/calculators/core:begin_loop_calculator",
|
||||
"//mediapipe/calculators/core:end_loop_calculator",
|
||||
"//mediapipe/calculators/util:annotation_overlay_calculator",
|
||||
"//mediapipe/calculators/util:detections_to_render_data_calculator",
|
||||
"//mediapipe/calculators/util:landmarks_to_render_data_calculator",
|
||||
"//mediapipe/calculators/util:rect_to_render_data_calculator",
|
||||
],
|
||||
)
|
||||
|
||||
mediapipe_simple_subgraph(
|
||||
name = "hand_detection_gpu",
|
||||
graph = "hand_detection_gpu.pbtxt",
|
||||
register_as = "HandDetectionSubgraph",
|
||||
deps = [
|
||||
"//mediapipe/calculators/image:image_properties_calculator",
|
||||
"//mediapipe/calculators/image:image_transformation_calculator",
|
||||
"//mediapipe/calculators/tflite:ssd_anchors_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_converter_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_custom_op_resolver_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_inference_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_tensors_to_detections_calculator",
|
||||
"//mediapipe/calculators/util:detection_label_id_to_text_calculator",
|
||||
"//mediapipe/calculators/util:detection_letterbox_removal_calculator",
|
||||
"//mediapipe/calculators/util:detections_to_rects_calculator",
|
||||
"//mediapipe/calculators/util:non_max_suppression_calculator",
|
||||
"//mediapipe/calculators/util:rect_transformation_calculator",
|
||||
],
|
||||
)
|
||||
|
||||
mediapipe_simple_subgraph(
|
||||
name = "multi_hand_detection_gpu",
|
||||
graph = "multi_hand_detection_gpu.pbtxt",
|
||||
register_as = "MultiHandDetectionSubgraph",
|
||||
deps = [
|
||||
"//mediapipe/calculators/core:begin_loop_calculator",
|
||||
"//mediapipe/calculators/core:clip_vector_size_calculator",
|
||||
"//mediapipe/calculators/core:end_loop_calculator",
|
||||
"//mediapipe/calculators/image:image_properties_calculator",
|
||||
"//mediapipe/calculators/image:image_transformation_calculator",
|
||||
"//mediapipe/calculators/tflite:ssd_anchors_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_converter_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_custom_op_resolver_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_inference_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_tensors_to_detections_calculator",
|
||||
"//mediapipe/calculators/util:detection_label_id_to_text_calculator",
|
||||
"//mediapipe/calculators/util:detection_letterbox_removal_calculator",
|
||||
"//mediapipe/calculators/util:detections_to_rects_calculator",
|
||||
"//mediapipe/calculators/util:non_max_suppression_calculator",
|
||||
"//mediapipe/calculators/util:rect_transformation_calculator",
|
||||
],
|
||||
)
|
||||
|
||||
mediapipe_simple_subgraph(
|
||||
name = "hand_landmark_gpu",
|
||||
graph = "hand_landmark_gpu.pbtxt",
|
||||
register_as = "HandLandmarkSubgraph",
|
||||
deps = [
|
||||
"//mediapipe/calculators/core:split_normalized_landmark_list_calculator",
|
||||
"//mediapipe/calculators/core:gate_calculator",
|
||||
"//mediapipe/calculators/core:split_vector_calculator",
|
||||
"//mediapipe/calculators/image:image_cropping_calculator",
|
||||
"//mediapipe/calculators/image:image_properties_calculator",
|
||||
"//mediapipe/calculators/image:image_transformation_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_converter_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_custom_op_resolver_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_inference_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_tensors_to_classification_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_tensors_to_floats_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_tensors_to_landmarks_calculator",
|
||||
"//mediapipe/calculators/util:detections_to_rects_calculator",
|
||||
"//mediapipe/calculators/util:landmark_letterbox_removal_calculator",
|
||||
"//mediapipe/calculators/util:landmark_projection_calculator",
|
||||
"//mediapipe/calculators/util:landmarks_to_detection_calculator",
|
||||
"//mediapipe/calculators/util:rect_transformation_calculator",
|
||||
"//mediapipe/calculators/util:thresholding_calculator",
|
||||
"//mediapipe/graphs/hand_tracking/calculators:hand_landmarks_to_rect_calculator",
|
||||
],
|
||||
)
|
||||
|
||||
mediapipe_simple_subgraph(
|
||||
name = "multi_hand_landmark_gpu",
|
||||
graph = "multi_hand_landmark.pbtxt",
|
||||
register_as = "MultiHandLandmarkSubgraph",
|
||||
deps = [
|
||||
":hand_landmark_gpu",
|
||||
"//mediapipe/calculators/core:begin_loop_calculator",
|
||||
"//mediapipe/calculators/core:end_loop_calculator",
|
||||
"//mediapipe/calculators/util:filter_collection_calculator",
|
||||
],
|
||||
)
|
||||
|
||||
mediapipe_simple_subgraph(
|
||||
name = "renderer_gpu",
|
||||
graph = "renderer_gpu.pbtxt",
|
||||
register_as = "RendererSubgraph",
|
||||
deps = [
|
||||
"//mediapipe/calculators/util:annotation_overlay_calculator",
|
||||
"//mediapipe/calculators/util:collection_has_min_size_calculator",
|
||||
"//mediapipe/calculators/util:detections_to_render_data_calculator",
|
||||
"//mediapipe/calculators/util:labels_to_render_data_calculator",
|
||||
"//mediapipe/calculators/util:landmarks_to_render_data_calculator",
|
||||
"//mediapipe/calculators/util:rect_to_render_data_calculator",
|
||||
],
|
||||
)
|
||||
|
||||
mediapipe_simple_subgraph(
|
||||
name = "multi_hand_renderer_gpu",
|
||||
graph = "multi_hand_renderer_gpu.pbtxt",
|
||||
register_as = "MultiHandRendererSubgraph",
|
||||
deps = [
|
||||
"//mediapipe/calculators/core:begin_loop_calculator",
|
||||
"//mediapipe/calculators/core:end_loop_calculator",
|
||||
"//mediapipe/calculators/util:annotation_overlay_calculator",
|
||||
"//mediapipe/calculators/util:detections_to_render_data_calculator",
|
||||
"//mediapipe/calculators/util:landmarks_to_render_data_calculator",
|
||||
"//mediapipe/calculators/util:rect_to_render_data_calculator",
|
||||
],
|
||||
)
|
||||
|
||||
@@ -1,193 +0,0 @@
|
||||
# MediaPipe hand detection subgraph.
|
||||
|
||||
type: "HandDetectionSubgraph"
|
||||
|
||||
input_stream: "input_video"
|
||||
output_stream: "DETECTIONS:palm_detections"
|
||||
output_stream: "NORM_RECT:hand_rect_from_palm_detections"
|
||||
|
||||
# Transforms the input image on CPU to a 256x256 image. To scale the input
|
||||
# image, the scale_mode option is set to FIT to preserve the aspect ratio,
|
||||
# resulting in potential letterboxing in the transformed image.
|
||||
node: {
|
||||
calculator: "ImageTransformationCalculator"
|
||||
input_stream: "IMAGE:input_video"
|
||||
output_stream: "IMAGE:transformed_input_video"
|
||||
output_stream: "LETTERBOX_PADDING:letterbox_padding"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.ImageTransformationCalculatorOptions] {
|
||||
output_width: 256
|
||||
output_height: 256
|
||||
scale_mode: FIT
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Generates a single side packet containing a TensorFlow Lite op resolver that
|
||||
# supports custom ops needed by the model used in this graph.
|
||||
node {
|
||||
calculator: "TfLiteCustomOpResolverCalculator"
|
||||
output_side_packet: "op_resolver"
|
||||
}
|
||||
|
||||
# Converts the transformed input image on CPU into an image tensor as a
|
||||
# TfLiteTensor. The zero_center option is set to true to normalize the
|
||||
# pixel values to [-1.f, 1.f] as opposed to [0.f, 1.f].
|
||||
node {
|
||||
calculator: "TfLiteConverterCalculator"
|
||||
input_stream: "IMAGE:transformed_input_video"
|
||||
output_stream: "TENSORS:image_tensor"
|
||||
}
|
||||
|
||||
# Runs a TensorFlow Lite model on CPU that takes an image tensor and outputs a
|
||||
# vector of tensors representing, for instance, detection boxes/keypoints and
|
||||
# scores.
|
||||
node {
|
||||
calculator: "TfLiteInferenceCalculator"
|
||||
input_stream: "TENSORS:image_tensor"
|
||||
output_stream: "TENSORS:detection_tensors"
|
||||
input_side_packet: "CUSTOM_OP_RESOLVER:op_resolver"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.TfLiteInferenceCalculatorOptions] {
|
||||
model_path: "mediapipe/models/palm_detection.tflite"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Generates a single side packet containing a vector of SSD anchors based on
|
||||
# the specification in the options.
|
||||
node {
|
||||
calculator: "SsdAnchorsCalculator"
|
||||
output_side_packet: "anchors"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.SsdAnchorsCalculatorOptions] {
|
||||
num_layers: 5
|
||||
min_scale: 0.1171875
|
||||
max_scale: 0.75
|
||||
input_size_height: 256
|
||||
input_size_width: 256
|
||||
anchor_offset_x: 0.5
|
||||
anchor_offset_y: 0.5
|
||||
strides: 8
|
||||
strides: 16
|
||||
strides: 32
|
||||
strides: 32
|
||||
strides: 32
|
||||
aspect_ratios: 1.0
|
||||
fixed_anchor_size: true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Decodes the detection tensors generated by the TensorFlow Lite model, based on
|
||||
# the SSD anchors and the specification in the options, into a vector of
|
||||
# detections. Each detection describes a detected object.
|
||||
node {
|
||||
calculator: "TfLiteTensorsToDetectionsCalculator"
|
||||
input_stream: "TENSORS:detection_tensors"
|
||||
input_side_packet: "ANCHORS:anchors"
|
||||
output_stream: "DETECTIONS:detections"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.TfLiteTensorsToDetectionsCalculatorOptions] {
|
||||
num_classes: 1
|
||||
num_boxes: 2944
|
||||
num_coords: 18
|
||||
box_coord_offset: 0
|
||||
keypoint_coord_offset: 4
|
||||
num_keypoints: 7
|
||||
num_values_per_keypoint: 2
|
||||
sigmoid_score: true
|
||||
score_clipping_thresh: 100.0
|
||||
reverse_output_order: true
|
||||
|
||||
x_scale: 256.0
|
||||
y_scale: 256.0
|
||||
h_scale: 256.0
|
||||
w_scale: 256.0
|
||||
min_score_thresh: 0.5
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Performs non-max suppression to remove excessive detections.
|
||||
node {
|
||||
calculator: "NonMaxSuppressionCalculator"
|
||||
input_stream: "detections"
|
||||
output_stream: "filtered_detections"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.NonMaxSuppressionCalculatorOptions] {
|
||||
min_suppression_threshold: 0.3
|
||||
min_score_threshold: 0.5
|
||||
overlap_type: INTERSECTION_OVER_UNION
|
||||
algorithm: WEIGHTED
|
||||
return_empty_detections: true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Maps detection label IDs to the corresponding label text. The label map is
|
||||
# provided in the label_map_path option.
|
||||
node {
|
||||
calculator: "DetectionLabelIdToTextCalculator"
|
||||
input_stream: "filtered_detections"
|
||||
output_stream: "labeled_detections"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.DetectionLabelIdToTextCalculatorOptions] {
|
||||
label_map_path: "mediapipe/models/palm_detection_labelmap.txt"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Adjusts detection locations (already normalized to [0.f, 1.f]) on the
|
||||
# letterboxed image (after image transformation with the FIT scale mode) to the
|
||||
# corresponding locations on the same image with the letterbox removed (the
|
||||
# input image to the graph before image transformation).
|
||||
node {
|
||||
calculator: "DetectionLetterboxRemovalCalculator"
|
||||
input_stream: "DETECTIONS:labeled_detections"
|
||||
input_stream: "LETTERBOX_PADDING:letterbox_padding"
|
||||
output_stream: "DETECTIONS:palm_detections"
|
||||
}
|
||||
|
||||
# Extracts image size from the input images.
|
||||
node {
|
||||
calculator: "ImagePropertiesCalculator"
|
||||
input_stream: "IMAGE:input_video"
|
||||
output_stream: "SIZE:image_size"
|
||||
}
|
||||
|
||||
# Converts results of palm detection into a rectangle (normalized by image size)
|
||||
# that encloses the palm and is rotated such that the line connecting center of
|
||||
# the wrist and MCP of the middle finger is aligned with the Y-axis of the
|
||||
# rectangle.
|
||||
node {
|
||||
calculator: "DetectionsToRectsCalculator"
|
||||
input_stream: "DETECTIONS:palm_detections"
|
||||
input_stream: "IMAGE_SIZE:image_size"
|
||||
output_stream: "NORM_RECT:palm_rect"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.DetectionsToRectsCalculatorOptions] {
|
||||
rotation_vector_start_keypoint_index: 0 # Center of wrist.
|
||||
rotation_vector_end_keypoint_index: 2 # MCP of middle finger.
|
||||
rotation_vector_target_angle_degrees: 90
|
||||
output_zero_rect_for_empty_detections: true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Expands and shifts the rectangle that contains the palm so that it's likely
|
||||
# to cover the entire hand.
|
||||
node {
|
||||
calculator: "RectTransformationCalculator"
|
||||
input_stream: "NORM_RECT:palm_rect"
|
||||
input_stream: "IMAGE_SIZE:image_size"
|
||||
output_stream: "hand_rect_from_palm_detections"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.RectTransformationCalculatorOptions] {
|
||||
scale_x: 2.6
|
||||
scale_y: 2.6
|
||||
shift_y: -0.5
|
||||
square_long: true
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,197 +0,0 @@
|
||||
# MediaPipe hand detection subgraph.
|
||||
|
||||
type: "HandDetectionSubgraph"
|
||||
|
||||
input_stream: "input_video"
|
||||
output_stream: "DETECTIONS:palm_detections"
|
||||
output_stream: "NORM_RECT:hand_rect_from_palm_detections"
|
||||
|
||||
# Transforms the input image on GPU to a 256x256 image. To scale the input
|
||||
# image, the scale_mode option is set to FIT to preserve the aspect ratio,
|
||||
# resulting in potential letterboxing in the transformed image.
|
||||
node: {
|
||||
calculator: "ImageTransformationCalculator"
|
||||
input_stream: "IMAGE_GPU:input_video"
|
||||
output_stream: "IMAGE_GPU:transformed_input_video"
|
||||
output_stream: "LETTERBOX_PADDING:letterbox_padding"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.ImageTransformationCalculatorOptions] {
|
||||
output_width: 256
|
||||
output_height: 256
|
||||
scale_mode: FIT
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Generates a single side packet containing a TensorFlow Lite op resolver that
|
||||
# supports custom ops needed by the model used in this graph.
|
||||
node {
|
||||
calculator: "TfLiteCustomOpResolverCalculator"
|
||||
output_side_packet: "opresolver"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.TfLiteCustomOpResolverCalculatorOptions] {
|
||||
use_gpu: true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Converts the transformed input image on GPU into an image tensor stored as a
|
||||
# TfLiteTensor.
|
||||
node {
|
||||
calculator: "TfLiteConverterCalculator"
|
||||
input_stream: "IMAGE_GPU:transformed_input_video"
|
||||
output_stream: "TENSORS_GPU:image_tensor"
|
||||
}
|
||||
|
||||
# Runs a TensorFlow Lite model on GPU that takes an image tensor and outputs a
|
||||
# vector of tensors representing, for instance, detection boxes/keypoints and
|
||||
# scores.
|
||||
node {
|
||||
calculator: "TfLiteInferenceCalculator"
|
||||
input_stream: "TENSORS_GPU:image_tensor"
|
||||
output_stream: "TENSORS_GPU:detection_tensors"
|
||||
input_side_packet: "CUSTOM_OP_RESOLVER:opresolver"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.TfLiteInferenceCalculatorOptions] {
|
||||
model_path: "mediapipe/models/palm_detection.tflite"
|
||||
use_gpu: true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Generates a single side packet containing a vector of SSD anchors based on
|
||||
# the specification in the options.
|
||||
node {
|
||||
calculator: "SsdAnchorsCalculator"
|
||||
output_side_packet: "anchors"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.SsdAnchorsCalculatorOptions] {
|
||||
num_layers: 5
|
||||
min_scale: 0.1171875
|
||||
max_scale: 0.75
|
||||
input_size_height: 256
|
||||
input_size_width: 256
|
||||
anchor_offset_x: 0.5
|
||||
anchor_offset_y: 0.5
|
||||
strides: 8
|
||||
strides: 16
|
||||
strides: 32
|
||||
strides: 32
|
||||
strides: 32
|
||||
aspect_ratios: 1.0
|
||||
fixed_anchor_size: true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Decodes the detection tensors generated by the TensorFlow Lite model, based on
|
||||
# the SSD anchors and the specification in the options, into a vector of
|
||||
# detections. Each detection describes a detected object.
|
||||
node {
|
||||
calculator: "TfLiteTensorsToDetectionsCalculator"
|
||||
input_stream: "TENSORS_GPU:detection_tensors"
|
||||
input_side_packet: "ANCHORS:anchors"
|
||||
output_stream: "DETECTIONS:detections"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.TfLiteTensorsToDetectionsCalculatorOptions] {
|
||||
num_classes: 1
|
||||
num_boxes: 2944
|
||||
num_coords: 18
|
||||
box_coord_offset: 0
|
||||
keypoint_coord_offset: 4
|
||||
num_keypoints: 7
|
||||
num_values_per_keypoint: 2
|
||||
sigmoid_score: true
|
||||
score_clipping_thresh: 100.0
|
||||
reverse_output_order: true
|
||||
|
||||
x_scale: 256.0
|
||||
y_scale: 256.0
|
||||
h_scale: 256.0
|
||||
w_scale: 256.0
|
||||
min_score_thresh: 0.7
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Performs non-max suppression to remove excessive detections.
|
||||
node {
|
||||
calculator: "NonMaxSuppressionCalculator"
|
||||
input_stream: "detections"
|
||||
output_stream: "filtered_detections"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.NonMaxSuppressionCalculatorOptions] {
|
||||
min_suppression_threshold: 0.3
|
||||
overlap_type: INTERSECTION_OVER_UNION
|
||||
algorithm: WEIGHTED
|
||||
return_empty_detections: true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Maps detection label IDs to the corresponding label text ("Palm"). The label
|
||||
# map is provided in the label_map_path option.
|
||||
node {
|
||||
calculator: "DetectionLabelIdToTextCalculator"
|
||||
input_stream: "filtered_detections"
|
||||
output_stream: "labeled_detections"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.DetectionLabelIdToTextCalculatorOptions] {
|
||||
label_map_path: "mediapipe/models/palm_detection_labelmap.txt"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Adjusts detection locations (already normalized to [0.f, 1.f]) on the
|
||||
# letterboxed image (after image transformation with the FIT scale mode) to the
|
||||
# corresponding locations on the same image with the letterbox removed (the
|
||||
# input image to the graph before image transformation).
|
||||
node {
|
||||
calculator: "DetectionLetterboxRemovalCalculator"
|
||||
input_stream: "DETECTIONS:labeled_detections"
|
||||
input_stream: "LETTERBOX_PADDING:letterbox_padding"
|
||||
output_stream: "DETECTIONS:palm_detections"
|
||||
}
|
||||
|
||||
# Extracts image size from the input images.
|
||||
node {
|
||||
calculator: "ImagePropertiesCalculator"
|
||||
input_stream: "IMAGE_GPU:input_video"
|
||||
output_stream: "SIZE:image_size"
|
||||
}
|
||||
|
||||
# Converts results of palm detection into a rectangle (normalized by image size)
|
||||
# that encloses the palm and is rotated such that the line connecting center of
|
||||
# the wrist and MCP of the middle finger is aligned with the Y-axis of the
|
||||
# rectangle.
|
||||
node {
|
||||
calculator: "DetectionsToRectsCalculator"
|
||||
input_stream: "DETECTIONS:palm_detections"
|
||||
input_stream: "IMAGE_SIZE:image_size"
|
||||
output_stream: "NORM_RECT:palm_rect"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.DetectionsToRectsCalculatorOptions] {
|
||||
rotation_vector_start_keypoint_index: 0 # Center of wrist.
|
||||
rotation_vector_end_keypoint_index: 2 # MCP of middle finger.
|
||||
rotation_vector_target_angle_degrees: 90
|
||||
output_zero_rect_for_empty_detections: true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Expands and shifts the rectangle that contains the palm so that it's likely
|
||||
# to cover the entire hand.
|
||||
node {
|
||||
calculator: "RectTransformationCalculator"
|
||||
input_stream: "NORM_RECT:palm_rect"
|
||||
input_stream: "IMAGE_SIZE:image_size"
|
||||
output_stream: "hand_rect_from_palm_detections"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.RectTransformationCalculatorOptions] {
|
||||
scale_x: 2.6
|
||||
scale_y: 2.6
|
||||
shift_y: -0.5
|
||||
square_long: true
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,226 +0,0 @@
|
||||
# MediaPipe hand landmark localization subgraph.
|
||||
|
||||
type: "HandLandmarkSubgraph"
|
||||
|
||||
input_stream: "IMAGE:input_video"
|
||||
input_stream: "NORM_RECT:hand_rect"
|
||||
output_stream: "LANDMARKS:hand_landmarks"
|
||||
output_stream: "NORM_RECT:hand_rect_for_next_frame"
|
||||
output_stream: "PRESENCE:hand_presence"
|
||||
output_stream: "PRESENCE_SCORE:hand_presence_score"
|
||||
output_stream: "HANDEDNESS:handedness"
|
||||
|
||||
# Crops the rectangle that contains a hand from the input image.
|
||||
node {
|
||||
calculator: "ImageCroppingCalculator"
|
||||
input_stream: "IMAGE:input_video"
|
||||
input_stream: "NORM_RECT:hand_rect"
|
||||
output_stream: "IMAGE:hand_image"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.ImageCroppingCalculatorOptions] {
|
||||
border_mode: BORDER_REPLICATE
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Transforms the input image on CPU to a 256x256 image. To scale the input
|
||||
# image, the scale_mode option is set to FIT to preserve the aspect ratio,
|
||||
# resulting in potential letterboxing in the transformed image.
|
||||
node: {
|
||||
calculator: "ImageTransformationCalculator"
|
||||
input_stream: "IMAGE:hand_image"
|
||||
output_stream: "IMAGE:transformed_input_video"
|
||||
output_stream: "LETTERBOX_PADDING:letterbox_padding"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.ImageTransformationCalculatorOptions] {
|
||||
output_width: 256
|
||||
output_height: 256
|
||||
scale_mode: FIT
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Generates a single side packet containing a TensorFlow Lite op resolver that
|
||||
# supports custom ops needed by the model used in this graph.
|
||||
node {
|
||||
calculator: "TfLiteCustomOpResolverCalculator"
|
||||
output_side_packet: "op_resolver"
|
||||
}
|
||||
|
||||
# Converts the transformed input image on CPU into an image tensor stored in
|
||||
# TfliteTensor. The zero_center option is set to false to normalize the
|
||||
# pixel values to [0.f, 1.f].
|
||||
node {
|
||||
calculator: "TfLiteConverterCalculator"
|
||||
input_stream: "IMAGE:transformed_input_video"
|
||||
output_stream: "TENSORS:image_tensor"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.TfLiteConverterCalculatorOptions] {
|
||||
zero_center: false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Runs a TensorFlow Lite model on CPU that takes an image tensor and outputs a
|
||||
# vector of tensors representing, for instance, detection boxes/keypoints and
|
||||
# scores.
|
||||
node {
|
||||
calculator: "TfLiteInferenceCalculator"
|
||||
input_stream: "TENSORS:image_tensor"
|
||||
output_stream: "TENSORS:output_tensors"
|
||||
input_side_packet: "CUSTOM_OP_RESOLVER:op_resolver"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.TfLiteInferenceCalculatorOptions] {
|
||||
model_path: "mediapipe/models/hand_landmark.tflite"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Splits a vector of TFLite tensors to multiple vectors according to the ranges
|
||||
# specified in option.
|
||||
node {
|
||||
calculator: "SplitTfLiteTensorVectorCalculator"
|
||||
input_stream: "output_tensors"
|
||||
output_stream: "landmark_tensors"
|
||||
output_stream: "hand_flag_tensor"
|
||||
output_stream: "handedness_tensor"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.SplitVectorCalculatorOptions] {
|
||||
ranges: { begin: 0 end: 1 }
|
||||
ranges: { begin: 1 end: 2 }
|
||||
ranges: { begin: 2 end: 3 }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Converts the hand-flag tensor into a float that represents the confidence
|
||||
# score of hand presence.
|
||||
node {
|
||||
calculator: "TfLiteTensorsToFloatsCalculator"
|
||||
input_stream: "TENSORS:hand_flag_tensor"
|
||||
output_stream: "FLOAT:hand_presence_score"
|
||||
}
|
||||
|
||||
# Converts the handedness tensor into a float as the score of the handedness
|
||||
# binary classifciation.
|
||||
node {
|
||||
calculator: "TfLiteTensorsToClassificationCalculator"
|
||||
input_stream: "TENSORS:handedness_tensor"
|
||||
output_stream: "CLASSIFICATIONS:handedness"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.TfLiteTensorsToClassificationCalculatorOptions] {
|
||||
top_k: 1
|
||||
label_map_path: "mediapipe/models/handedness.txt"
|
||||
binary_classification: true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Applies a threshold to the confidence score to determine whether a hand is
|
||||
# present.
|
||||
node {
|
||||
calculator: "ThresholdingCalculator"
|
||||
input_stream: "FLOAT:hand_presence_score"
|
||||
output_stream: "FLAG:hand_presence"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.ThresholdingCalculatorOptions] {
|
||||
threshold: 0.5
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Decodes the landmark tensors into a list of landmarks, where the landmark
|
||||
# coordinates are normalized by the size of the input image to the model.
|
||||
node {
|
||||
calculator: "TfLiteTensorsToLandmarksCalculator"
|
||||
input_stream: "TENSORS:landmark_tensors"
|
||||
output_stream: "NORM_LANDMARKS:landmarks"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.TfLiteTensorsToLandmarksCalculatorOptions] {
|
||||
num_landmarks: 21
|
||||
input_image_width: 256
|
||||
input_image_height: 256
|
||||
# The additional scaling factor is used to account for the Z coordinate
|
||||
# distribution in the training data.
|
||||
normalize_z: 0.4
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Adjusts landmarks (already normalized to [0.f, 1.f]) on the letterboxed hand
|
||||
# image (after image transformation with the FIT scale mode) to the
|
||||
# corresponding locations on the same image with the letterbox removed (hand
|
||||
# image before image transformation).
|
||||
node {
|
||||
calculator: "LandmarkLetterboxRemovalCalculator"
|
||||
input_stream: "LANDMARKS:landmarks"
|
||||
input_stream: "LETTERBOX_PADDING:letterbox_padding"
|
||||
output_stream: "LANDMARKS:scaled_landmarks"
|
||||
}
|
||||
|
||||
# Projects the landmarks from the cropped hand image to the corresponding
|
||||
# locations on the full image before cropping (input to the graph).
|
||||
node {
|
||||
calculator: "LandmarkProjectionCalculator"
|
||||
input_stream: "NORM_LANDMARKS:scaled_landmarks"
|
||||
input_stream: "NORM_RECT:hand_rect"
|
||||
output_stream: "NORM_LANDMARKS:hand_landmarks"
|
||||
}
|
||||
|
||||
# Extracts image size from the input images.
|
||||
node {
|
||||
calculator: "ImagePropertiesCalculator"
|
||||
input_stream: "IMAGE:input_video"
|
||||
output_stream: "SIZE:image_size"
|
||||
}
|
||||
|
||||
# Extracts a subset of the hand landmarks that are relatively more stable across
|
||||
# frames (e.g. comparing to finger tips) for computing the bounding box. The box
|
||||
# will later be expanded to contain the entire hand. In this approach, it is
|
||||
# more robust to drastically changing hand size.
|
||||
# The landmarks extracted are: wrist, MCP/PIP of five fingers.
|
||||
node {
|
||||
calculator: "SplitNormalizedLandmarkListCalculator"
|
||||
input_stream: "hand_landmarks"
|
||||
output_stream: "partial_landmarks"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.SplitVectorCalculatorOptions] {
|
||||
ranges: { begin: 0 end: 4 }
|
||||
ranges: { begin: 5 end: 7 }
|
||||
ranges: { begin: 9 end: 11 }
|
||||
ranges: { begin: 13 end: 15 }
|
||||
ranges: { begin: 17 end: 19 }
|
||||
combine_outputs: true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Converts the hand landmarks into a rectangle (normalized by image size)
|
||||
# that encloses the hand. The calculator uses a subset of all hand landmarks
|
||||
# extracted from SplitNormalizedLandmarkListCalculator above to
|
||||
# calculate the bounding box and the rotation of the output rectangle. Please
|
||||
# see the comments in the calculator for more detail.
|
||||
node {
|
||||
calculator: "HandLandmarksToRectCalculator"
|
||||
input_stream: "NORM_LANDMARKS:partial_landmarks"
|
||||
input_stream: "IMAGE_SIZE:image_size"
|
||||
output_stream: "NORM_RECT:hand_rect_from_landmarks"
|
||||
}
|
||||
|
||||
# Expands the hand rectangle so that the box contains the entire hand and it's
|
||||
# big enough so that it's likely to still contain the hand even with some motion
|
||||
# in the next video frame .
|
||||
node {
|
||||
calculator: "RectTransformationCalculator"
|
||||
input_stream: "NORM_RECT:hand_rect_from_landmarks"
|
||||
input_stream: "IMAGE_SIZE:image_size"
|
||||
output_stream: "hand_rect_for_next_frame"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.RectTransformationCalculatorOptions] {
|
||||
scale_x: 2.0
|
||||
scale_y: 2.0
|
||||
shift_y: -0.1
|
||||
square_long: true
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,230 +0,0 @@
|
||||
# MediaPipe hand landmark localization subgraph.
|
||||
|
||||
type: "HandLandmarkSubgraph"
|
||||
|
||||
input_stream: "IMAGE:input_video"
|
||||
input_stream: "NORM_RECT:hand_rect"
|
||||
output_stream: "LANDMARKS:hand_landmarks"
|
||||
output_stream: "NORM_RECT:hand_rect_for_next_frame"
|
||||
output_stream: "PRESENCE:hand_presence"
|
||||
output_stream: "PRESENCE_SCORE:hand_presence_score"
|
||||
output_stream: "HANDEDNESS:handedness"
|
||||
|
||||
# Crops the rectangle that contains a hand from the input image.
|
||||
node {
|
||||
calculator: "ImageCroppingCalculator"
|
||||
input_stream: "IMAGE_GPU:input_video"
|
||||
input_stream: "NORM_RECT:hand_rect"
|
||||
output_stream: "IMAGE_GPU:hand_image"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.ImageCroppingCalculatorOptions] {
|
||||
border_mode: BORDER_REPLICATE
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Transforms the input image on GPU to a 256x256 image. To scale the input
|
||||
# image, the scale_mode option is set to FIT to preserve the aspect ratio,
|
||||
# resulting in potential letterboxing in the transformed image.
|
||||
node: {
|
||||
calculator: "ImageTransformationCalculator"
|
||||
input_stream: "IMAGE_GPU:hand_image"
|
||||
output_stream: "IMAGE_GPU:transformed_hand_image"
|
||||
output_stream: "LETTERBOX_PADDING:letterbox_padding"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.ImageTransformationCalculatorOptions] {
|
||||
output_width: 256
|
||||
output_height: 256
|
||||
scale_mode: FIT
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Generates a single side packet containing a TensorFlow Lite op resolver that
|
||||
# supports custom ops needed by the model used in this graph.
|
||||
node {
|
||||
calculator: "TfLiteCustomOpResolverCalculator"
|
||||
output_side_packet: "op_resolver"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.TfLiteCustomOpResolverCalculatorOptions] {
|
||||
use_gpu: true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Converts the transformed input image on GPU into an image tensor stored as a
|
||||
# TfLiteTensor.
|
||||
node {
|
||||
calculator: "TfLiteConverterCalculator"
|
||||
input_stream: "IMAGE_GPU:transformed_hand_image"
|
||||
output_stream: "TENSORS_GPU:image_tensor"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.TfLiteConverterCalculatorOptions] {
|
||||
zero_center: false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Runs a TensorFlow Lite model on GPU that takes an image tensor and outputs a
|
||||
# vector of tensors representing, for instance, detection boxes/keypoints and
|
||||
# scores.
|
||||
node {
|
||||
calculator: "TfLiteInferenceCalculator"
|
||||
input_stream: "TENSORS_GPU:image_tensor"
|
||||
output_stream: "TENSORS:output_tensors"
|
||||
input_side_packet: "CUSTOM_OP_RESOLVER:op_resolver"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.TfLiteInferenceCalculatorOptions] {
|
||||
model_path: "mediapipe/models/hand_landmark.tflite"
|
||||
use_gpu: true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Splits a vector of tensors into multiple vectors.
|
||||
node {
|
||||
calculator: "SplitTfLiteTensorVectorCalculator"
|
||||
input_stream: "output_tensors"
|
||||
output_stream: "landmark_tensors"
|
||||
output_stream: "hand_flag_tensor"
|
||||
output_stream: "handedness_tensor"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.SplitVectorCalculatorOptions] {
|
||||
ranges: { begin: 0 end: 1 }
|
||||
ranges: { begin: 1 end: 2 }
|
||||
ranges: { begin: 2 end: 3 }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Converts the hand-flag tensor into a float that represents the confidence
|
||||
# score of hand presence.
|
||||
node {
|
||||
calculator: "TfLiteTensorsToFloatsCalculator"
|
||||
input_stream: "TENSORS:hand_flag_tensor"
|
||||
output_stream: "FLOAT:hand_presence_score"
|
||||
}
|
||||
|
||||
# Converts the handedness tensor into a float as the score of the handedness
|
||||
# binary classifciation.
|
||||
node {
|
||||
calculator: "TfLiteTensorsToClassificationCalculator"
|
||||
input_stream: "TENSORS:handedness_tensor"
|
||||
output_stream: "CLASSIFICATIONS:handedness"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.TfLiteTensorsToClassificationCalculatorOptions] {
|
||||
top_k: 1
|
||||
label_map_path: "mediapipe/models/handedness.txt"
|
||||
binary_classification: true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Applies a threshold to the confidence score to determine whether a hand is
|
||||
# present.
|
||||
node {
|
||||
calculator: "ThresholdingCalculator"
|
||||
input_stream: "FLOAT:hand_presence_score"
|
||||
output_stream: "FLAG:hand_presence"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.ThresholdingCalculatorOptions] {
|
||||
threshold: 0.5
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Decodes the landmark tensors into a list of landmarks, where the landmark
|
||||
# coordinates are normalized by the size of the input image to the model.
|
||||
node {
|
||||
calculator: "TfLiteTensorsToLandmarksCalculator"
|
||||
input_stream: "TENSORS:landmark_tensors"
|
||||
output_stream: "NORM_LANDMARKS:landmarks"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.TfLiteTensorsToLandmarksCalculatorOptions] {
|
||||
num_landmarks: 21
|
||||
input_image_width: 256
|
||||
input_image_height: 256
|
||||
# The additional scaling factor is used to account for the Z coordinate
|
||||
# distribution in the training data.
|
||||
normalize_z: 0.4
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Adjusts landmarks (already normalized to [0.f, 1.f]) on the letterboxed hand
|
||||
# image (after image transformation with the FIT scale mode) to the
|
||||
# corresponding locations on the same image with the letterbox removed (hand
|
||||
# image before image transformation).
|
||||
node {
|
||||
calculator: "LandmarkLetterboxRemovalCalculator"
|
||||
input_stream: "LANDMARKS:landmarks"
|
||||
input_stream: "LETTERBOX_PADDING:letterbox_padding"
|
||||
output_stream: "LANDMARKS:scaled_landmarks"
|
||||
}
|
||||
|
||||
# Projects the landmarks from the cropped hand image to the corresponding
|
||||
# locations on the full image before cropping (input to the graph).
|
||||
node {
|
||||
calculator: "LandmarkProjectionCalculator"
|
||||
input_stream: "NORM_LANDMARKS:scaled_landmarks"
|
||||
input_stream: "NORM_RECT:hand_rect"
|
||||
output_stream: "NORM_LANDMARKS:hand_landmarks"
|
||||
}
|
||||
|
||||
# Extracts image size from the input images.
|
||||
node {
|
||||
calculator: "ImagePropertiesCalculator"
|
||||
input_stream: "IMAGE_GPU:input_video"
|
||||
output_stream: "SIZE:image_size"
|
||||
}
|
||||
|
||||
# Extracts a subset of the hand landmarks that are relatively more stable across
|
||||
# frames (e.g. comparing to finger tips) for computing the bounding box. The box
|
||||
# will later be expanded to contain the entire hand. In this approach, it is
|
||||
# more robust to drastically changing hand size.
|
||||
# The landmarks extracted are: wrist, MCP/PIP of five fingers.
|
||||
node {
|
||||
calculator: "SplitNormalizedLandmarkListCalculator"
|
||||
input_stream: "hand_landmarks"
|
||||
output_stream: "partial_landmarks"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.SplitVectorCalculatorOptions] {
|
||||
ranges: { begin: 0 end: 4 }
|
||||
ranges: { begin: 5 end: 7 }
|
||||
ranges: { begin: 9 end: 11 }
|
||||
ranges: { begin: 13 end: 15 }
|
||||
ranges: { begin: 17 end: 19 }
|
||||
combine_outputs: true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Converts the hand landmarks into a rectangle (normalized by image size)
|
||||
# that encloses the hand. The calculator uses a subset of all hand landmarks
|
||||
# extracted from SplitNormalizedLandmarkListCalculator above to
|
||||
# calculate the bounding box and the rotation of the output rectangle. Please
|
||||
# see the comments in the calculator for more detail.
|
||||
node {
|
||||
calculator: "HandLandmarksToRectCalculator"
|
||||
input_stream: "NORM_LANDMARKS:partial_landmarks"
|
||||
input_stream: "IMAGE_SIZE:image_size"
|
||||
output_stream: "NORM_RECT:hand_rect_from_landmarks"
|
||||
}
|
||||
|
||||
# Expands the hand rectangle so that the box contains the entire hand and it's
|
||||
# big enough so that it's likely to still contain the hand even with some motion
|
||||
# in the next video frame .
|
||||
node {
|
||||
calculator: "RectTransformationCalculator"
|
||||
input_stream: "NORM_RECT:hand_rect_from_landmarks"
|
||||
input_stream: "IMAGE_SIZE:image_size"
|
||||
output_stream: "hand_rect_for_next_frame"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.RectTransformationCalculatorOptions] {
|
||||
scale_x: 2.0
|
||||
scale_y: 2.0
|
||||
shift_y: -0.1
|
||||
square_long: true
|
||||
}
|
||||
}
|
||||
}
|
||||
+71
-6
@@ -1,16 +1,25 @@
|
||||
# MediaPipe multi-hand tracking rendering subgraph.
|
||||
# MediaPipe graph to render hand landmarks and some related debug information.
|
||||
|
||||
type: "MultiHandRendererSubgraph"
|
||||
type: "HandRendererSubgraph"
|
||||
|
||||
# CPU image. (ImageFrame)
|
||||
input_stream: "IMAGE:input_image"
|
||||
# A vector of NormalizedLandmarks, one for each hand.
|
||||
# Collection of detected/predicted hands, each represented as a list of
|
||||
# landmarks. (std::vector<NormalizedLandmarkList>)
|
||||
input_stream: "LANDMARKS:multi_hand_landmarks"
|
||||
# A vector of NormalizedRect, one for each hand.
|
||||
# Handedness of the detected hand (i.e. is hand left or right).
|
||||
# (std::vector<ClassificationList>)
|
||||
input_stream: "HANDEDNESS:multi_handedness"
|
||||
# Regions of interest calculated based on palm detections.
|
||||
# (std::vector<NormalizedRect>)
|
||||
input_stream: "NORM_RECTS:0:multi_palm_rects"
|
||||
# A vector of NormalizedRect, one for each hand.
|
||||
# Regions of interest calculated based on landmarks.
|
||||
# (std::vector<NormalizedRect>)
|
||||
input_stream: "NORM_RECTS:1:multi_hand_rects"
|
||||
# A vector of Detection, one for each hand.
|
||||
# Detected palms. (std::vector<Detection>)
|
||||
input_stream: "DETECTIONS:palm_detections"
|
||||
|
||||
# Updated CPU image. (ImageFrame)
|
||||
output_stream: "IMAGE:output_image"
|
||||
|
||||
# Converts detections to drawing primitives for annotation overlay.
|
||||
@@ -131,6 +140,61 @@ node {
|
||||
output_stream: "ITERABLE:multi_hand_landmarks_render_data"
|
||||
}
|
||||
|
||||
# Don't render handedness if there are more than one handedness reported.
|
||||
node {
|
||||
calculator: "ClassificationListVectorHasMinSizeCalculator"
|
||||
input_stream: "ITERABLE:multi_handedness"
|
||||
output_stream: "disallow_handedness_rendering"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.CollectionHasMinSizeCalculatorOptions] {
|
||||
min_size: 2
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
node {
|
||||
calculator: "GateCalculator"
|
||||
input_stream: "multi_handedness"
|
||||
input_stream: "DISALLOW:disallow_handedness_rendering"
|
||||
output_stream: "allowed_multi_handedness"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.GateCalculatorOptions] {
|
||||
empty_packets_as_allow: false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
node {
|
||||
calculator: "SplitClassificationListVectorCalculator"
|
||||
input_stream: "allowed_multi_handedness"
|
||||
output_stream: "handedness"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.SplitVectorCalculatorOptions] {
|
||||
ranges: { begin: 0 end: 1 }
|
||||
element_only: true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Converts classification to drawing primitives for annotation overlay.
|
||||
node {
|
||||
calculator: "LabelsToRenderDataCalculator"
|
||||
input_stream: "CLASSIFICATIONS:handedness"
|
||||
output_stream: "RENDER_DATA:handedness_render_data"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.LabelsToRenderDataCalculatorOptions]: {
|
||||
color { r: 255 g: 0 b: 0 }
|
||||
thickness: 10.0
|
||||
font_height_px: 50
|
||||
horizontal_offset_px: 30
|
||||
vertical_offset_px: 50
|
||||
|
||||
max_num_labels: 1
|
||||
location: TOP_LEFT
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Draws annotations and overlays them on top of the input images. Consumes
|
||||
# a vector of RenderData objects and draws each of them on the input frame.
|
||||
node {
|
||||
@@ -139,6 +203,7 @@ node {
|
||||
input_stream: "detection_render_data"
|
||||
input_stream: "multi_hand_rects_render_data"
|
||||
input_stream: "multi_palm_rects_render_data"
|
||||
input_stream: "handedness_render_data"
|
||||
input_stream: "VECTOR:0:multi_hand_landmarks_render_data"
|
||||
output_stream: "IMAGE:output_image"
|
||||
}
|
||||
+71
-6
@@ -1,16 +1,25 @@
|
||||
# MediaPipe multi-hand tracking rendering subgraph.
|
||||
# MediaPipe graph to render hand landmarks and some related debug information.
|
||||
|
||||
type: "MultiHandRendererSubgraph"
|
||||
type: "HandRendererSubgraph"
|
||||
|
||||
# GPU buffer. (GpuBuffer)
|
||||
input_stream: "IMAGE:input_image"
|
||||
# A vector of NormalizedLandmarks, one for each hand.
|
||||
# Collection of detected/predicted hands, each represented as a list of
|
||||
# landmarks. (std::vector<NormalizedLandmarkList>)
|
||||
input_stream: "LANDMARKS:multi_hand_landmarks"
|
||||
# A vector of NormalizedRect, one for each hand.
|
||||
# Handedness of the detected hand (i.e. is hand left or right).
|
||||
# (std::vector<ClassificationList>)
|
||||
input_stream: "HANDEDNESS:multi_handedness"
|
||||
# Regions of interest calculated based on palm detections.
|
||||
# (std::vector<NormalizedRect>)
|
||||
input_stream: "NORM_RECTS:0:multi_palm_rects"
|
||||
# A vector of NormalizedRect, one for each hand.
|
||||
# Regions of interest calculated based on landmarks.
|
||||
# (std::vector<NormalizedRect>)
|
||||
input_stream: "NORM_RECTS:1:multi_hand_rects"
|
||||
# A vector of Detection, one for each hand.
|
||||
# Detected palms. (std::vector<Detection>)
|
||||
input_stream: "DETECTIONS:palm_detections"
|
||||
|
||||
# Updated GPU buffer. (GpuBuffer)
|
||||
output_stream: "IMAGE:output_image"
|
||||
|
||||
# Converts detections to drawing primitives for annotation overlay.
|
||||
@@ -131,6 +140,61 @@ node {
|
||||
output_stream: "ITERABLE:multi_hand_landmarks_render_data"
|
||||
}
|
||||
|
||||
# Don't render handedness if there are more than one handedness reported.
|
||||
node {
|
||||
calculator: "ClassificationListVectorHasMinSizeCalculator"
|
||||
input_stream: "ITERABLE:multi_handedness"
|
||||
output_stream: "disallow_handedness_rendering"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.CollectionHasMinSizeCalculatorOptions] {
|
||||
min_size: 2
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
node {
|
||||
calculator: "GateCalculator"
|
||||
input_stream: "multi_handedness"
|
||||
input_stream: "DISALLOW:disallow_handedness_rendering"
|
||||
output_stream: "allowed_multi_handedness"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.GateCalculatorOptions] {
|
||||
empty_packets_as_allow: false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
node {
|
||||
calculator: "SplitClassificationListVectorCalculator"
|
||||
input_stream: "allowed_multi_handedness"
|
||||
output_stream: "handedness"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.SplitVectorCalculatorOptions] {
|
||||
ranges: { begin: 0 end: 1 }
|
||||
element_only: true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Converts classification to drawing primitives for annotation overlay.
|
||||
node {
|
||||
calculator: "LabelsToRenderDataCalculator"
|
||||
input_stream: "CLASSIFICATIONS:handedness"
|
||||
output_stream: "RENDER_DATA:handedness_render_data"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.LabelsToRenderDataCalculatorOptions]: {
|
||||
color { r: 255 g: 0 b: 0 }
|
||||
thickness: 10.0
|
||||
font_height_px: 50
|
||||
horizontal_offset_px: 30
|
||||
vertical_offset_px: 50
|
||||
|
||||
max_num_labels: 1
|
||||
location: TOP_LEFT
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Draws annotations and overlays them on top of the input images. Consumes
|
||||
# a vector of RenderData objects and draws each of them on the input frame.
|
||||
node {
|
||||
@@ -139,6 +203,7 @@ node {
|
||||
input_stream: "detection_render_data"
|
||||
input_stream: "multi_hand_rects_render_data"
|
||||
input_stream: "multi_palm_rects_render_data"
|
||||
input_stream: "handedness_render_data"
|
||||
input_stream: "VECTOR:0:multi_hand_landmarks_render_data"
|
||||
output_stream: "IMAGE_GPU:output_image"
|
||||
}
|
||||
@@ -1,212 +0,0 @@
|
||||
# MediaPipe multi-hand detection subgraph.
|
||||
|
||||
type: "MultiHandDetectionSubgraph"
|
||||
|
||||
input_stream: "input_video"
|
||||
output_stream: "DETECTIONS:palm_detections"
|
||||
output_stream: "NORM_RECTS:clipped_hand_rects_from_palm_detections"
|
||||
|
||||
# Transforms the input image on CPU to a 256x256 image. To scale the input
|
||||
# image, the scale_mode option is set to FIT to preserve the aspect ratio,
|
||||
# resulting in potential letterboxing in the transformed image.
|
||||
node: {
|
||||
calculator: "ImageTransformationCalculator"
|
||||
input_stream: "IMAGE:input_video"
|
||||
output_stream: "IMAGE:transformed_input_video"
|
||||
output_stream: "LETTERBOX_PADDING:letterbox_padding"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.ImageTransformationCalculatorOptions] {
|
||||
output_width: 256
|
||||
output_height: 256
|
||||
scale_mode: FIT
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Generates a single side packet containing a TensorFlow Lite op resolver that
|
||||
# supports custom ops needed by the model used in this graph.
|
||||
node {
|
||||
calculator: "TfLiteCustomOpResolverCalculator"
|
||||
output_side_packet: "opresolver"
|
||||
}
|
||||
|
||||
# Converts the transformed input image on CPU into an image tensor as a
|
||||
# TfLiteTensor. The zero_center option is set to true to normalize the
|
||||
# pixel values to [-1.f, 1.f] as opposed to [0.f, 1.f].
|
||||
node {
|
||||
calculator: "TfLiteConverterCalculator"
|
||||
input_stream: "IMAGE:transformed_input_video"
|
||||
output_stream: "TENSORS:image_tensor"
|
||||
}
|
||||
|
||||
# Runs a TensorFlow Lite model on CPU that takes an image tensor and outputs a
|
||||
# vector of tensors representing, for instance, detection boxes/keypoints and
|
||||
# scores.
|
||||
node {
|
||||
calculator: "TfLiteInferenceCalculator"
|
||||
input_stream: "TENSORS:image_tensor"
|
||||
output_stream: "TENSORS:detection_tensors"
|
||||
input_side_packet: "CUSTOM_OP_RESOLVER:opresolver"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.TfLiteInferenceCalculatorOptions] {
|
||||
model_path: "mediapipe/models/palm_detection.tflite"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Generates a single side packet containing a vector of SSD anchors based on
|
||||
# the specification in the options.
|
||||
node {
|
||||
calculator: "SsdAnchorsCalculator"
|
||||
output_side_packet: "anchors"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.SsdAnchorsCalculatorOptions] {
|
||||
num_layers: 5
|
||||
min_scale: 0.1171875
|
||||
max_scale: 0.75
|
||||
input_size_height: 256
|
||||
input_size_width: 256
|
||||
anchor_offset_x: 0.5
|
||||
anchor_offset_y: 0.5
|
||||
strides: 8
|
||||
strides: 16
|
||||
strides: 32
|
||||
strides: 32
|
||||
strides: 32
|
||||
aspect_ratios: 1.0
|
||||
fixed_anchor_size: true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Decodes the detection tensors generated by the TensorFlow Lite model, based on
|
||||
# the SSD anchors and the specification in the options, into a vector of
|
||||
# detections. Each detection describes a detected object.
|
||||
node {
|
||||
calculator: "TfLiteTensorsToDetectionsCalculator"
|
||||
input_stream: "TENSORS:detection_tensors"
|
||||
input_side_packet: "ANCHORS:anchors"
|
||||
output_stream: "DETECTIONS:detections"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.TfLiteTensorsToDetectionsCalculatorOptions] {
|
||||
num_classes: 1
|
||||
num_boxes: 2944
|
||||
num_coords: 18
|
||||
box_coord_offset: 0
|
||||
keypoint_coord_offset: 4
|
||||
num_keypoints: 7
|
||||
num_values_per_keypoint: 2
|
||||
sigmoid_score: true
|
||||
score_clipping_thresh: 100.0
|
||||
reverse_output_order: true
|
||||
|
||||
x_scale: 256.0
|
||||
y_scale: 256.0
|
||||
h_scale: 256.0
|
||||
w_scale: 256.0
|
||||
min_score_thresh: 0.7
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Performs non-max suppression to remove excessive detections.
|
||||
node {
|
||||
calculator: "NonMaxSuppressionCalculator"
|
||||
input_stream: "detections"
|
||||
output_stream: "filtered_detections"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.NonMaxSuppressionCalculatorOptions] {
|
||||
min_suppression_threshold: 0.3
|
||||
overlap_type: INTERSECTION_OVER_UNION
|
||||
algorithm: WEIGHTED
|
||||
return_empty_detections: true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Maps detection label IDs to the corresponding label text ("Palm"). The label
|
||||
# map is provided in the label_map_path option.
|
||||
node {
|
||||
calculator: "DetectionLabelIdToTextCalculator"
|
||||
input_stream: "filtered_detections"
|
||||
output_stream: "labeled_detections"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.DetectionLabelIdToTextCalculatorOptions] {
|
||||
label_map_path: "mediapipe/models/palm_detection_labelmap.txt"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Adjusts detection locations (already normalized to [0.f, 1.f]) on the
|
||||
# letterboxed image (after image transformation with the FIT scale mode) to the
|
||||
# corresponding locations on the same image with the letterbox removed (the
|
||||
# input image to the graph before image transformation).
|
||||
node {
|
||||
calculator: "DetectionLetterboxRemovalCalculator"
|
||||
input_stream: "DETECTIONS:labeled_detections"
|
||||
input_stream: "LETTERBOX_PADDING:letterbox_padding"
|
||||
output_stream: "DETECTIONS:palm_detections"
|
||||
}
|
||||
|
||||
# Extracts image size from the input images.
|
||||
node {
|
||||
calculator: "ImagePropertiesCalculator"
|
||||
input_stream: "IMAGE:input_video"
|
||||
output_stream: "SIZE:image_size"
|
||||
}
|
||||
|
||||
# Converts each palm detection into a rectangle (normalized by image size)
|
||||
# that encloses the palm and is rotated such that the line connecting center of
|
||||
# the wrist and MCP of the middle finger is aligned with the Y-axis of the
|
||||
# rectangle.
|
||||
node {
|
||||
calculator: "DetectionsToRectsCalculator"
|
||||
input_stream: "DETECTIONS:palm_detections"
|
||||
input_stream: "IMAGE_SIZE:image_size"
|
||||
output_stream: "NORM_RECTS:palm_rects"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.DetectionsToRectsCalculatorOptions] {
|
||||
rotation_vector_start_keypoint_index: 0 # Center of wrist.
|
||||
rotation_vector_end_keypoint_index: 2 # MCP of middle finger.
|
||||
rotation_vector_target_angle_degrees: 90
|
||||
output_zero_rect_for_empty_detections: true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Expands and shifts the rectangle that contains the palm so that it's likely
|
||||
# to cover the entire hand.
|
||||
node {
|
||||
calculator: "RectTransformationCalculator"
|
||||
input_stream: "NORM_RECTS:palm_rects"
|
||||
input_stream: "IMAGE_SIZE:image_size"
|
||||
output_stream: "hand_rects_from_palm_detections"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.RectTransformationCalculatorOptions] {
|
||||
scale_x: 2.6
|
||||
scale_y: 2.6
|
||||
shift_y: -0.5
|
||||
square_long: true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Clips the size of the input vector to the provided max_vec_size. This
|
||||
# determines the maximum number of hand instances this graph outputs.
|
||||
# Note that the performance gain of clipping detections earlier in this graph is
|
||||
# minimal because NMS will minimize overlapping detections and the number of
|
||||
# detections isn't expected to exceed 5-10.
|
||||
node {
|
||||
calculator: "ClipNormalizedRectVectorSizeCalculator"
|
||||
input_stream: "hand_rects_from_palm_detections"
|
||||
output_stream: "clipped_hand_rects_from_palm_detections"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.ClipVectorSizeCalculatorOptions] {
|
||||
# This value can be changed to support tracking arbitrary number of hands.
|
||||
# Please also remember to modify min_size in
|
||||
# CollectionHsMinSizeCalculatorOptions in
|
||||
# mediapipe/graphs/hand_tracking/multi_hand_tracking_desktop.pbtxt.
|
||||
max_vec_size: 2
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,218 +0,0 @@
|
||||
# MediaPipe multi-hand detection subgraph.
|
||||
|
||||
type: "MultiHandDetectionSubgraph"
|
||||
|
||||
input_stream: "input_video"
|
||||
output_stream: "DETECTIONS:palm_detections"
|
||||
output_stream: "NORM_RECTS:clipped_hand_rects_from_palm_detections"
|
||||
|
||||
# Transforms the input image on GPU to a 256x256 image. To scale the input
|
||||
# image, the scale_mode option is set to FIT to preserve the aspect ratio,
|
||||
# resulting in potential letterboxing in the transformed image.
|
||||
node: {
|
||||
calculator: "ImageTransformationCalculator"
|
||||
input_stream: "IMAGE_GPU:input_video"
|
||||
output_stream: "IMAGE_GPU:transformed_input_video"
|
||||
output_stream: "LETTERBOX_PADDING:letterbox_padding"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.ImageTransformationCalculatorOptions] {
|
||||
output_width: 256
|
||||
output_height: 256
|
||||
scale_mode: FIT
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Generates a single side packet containing a TensorFlow Lite op resolver that
|
||||
# supports custom ops needed by the model used in this graph.
|
||||
node {
|
||||
calculator: "TfLiteCustomOpResolverCalculator"
|
||||
output_side_packet: "opresolver"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.TfLiteCustomOpResolverCalculatorOptions] {
|
||||
use_gpu: true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Converts the transformed input image on GPU into an image tensor stored as a
|
||||
# TfLiteTensor.
|
||||
node {
|
||||
calculator: "TfLiteConverterCalculator"
|
||||
input_stream: "IMAGE_GPU:transformed_input_video"
|
||||
output_stream: "TENSORS_GPU:image_tensor"
|
||||
}
|
||||
|
||||
# Runs a TensorFlow Lite model on GPU that takes an image tensor and outputs a
|
||||
# vector of tensors representing, for instance, detection boxes/keypoints and
|
||||
# scores.
|
||||
node {
|
||||
calculator: "TfLiteInferenceCalculator"
|
||||
input_stream: "TENSORS_GPU:image_tensor"
|
||||
output_stream: "TENSORS_GPU:detection_tensors"
|
||||
input_side_packet: "CUSTOM_OP_RESOLVER:opresolver"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.TfLiteInferenceCalculatorOptions] {
|
||||
model_path: "mediapipe/models/palm_detection.tflite"
|
||||
use_gpu: true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Generates a single side packet containing a vector of SSD anchors based on
|
||||
# the specification in the options.
|
||||
node {
|
||||
calculator: "SsdAnchorsCalculator"
|
||||
output_side_packet: "anchors"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.SsdAnchorsCalculatorOptions] {
|
||||
num_layers: 5
|
||||
min_scale: 0.1171875
|
||||
max_scale: 0.75
|
||||
input_size_height: 256
|
||||
input_size_width: 256
|
||||
anchor_offset_x: 0.5
|
||||
anchor_offset_y: 0.5
|
||||
strides: 8
|
||||
strides: 16
|
||||
strides: 32
|
||||
strides: 32
|
||||
strides: 32
|
||||
aspect_ratios: 1.0
|
||||
fixed_anchor_size: true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Decodes the detection tensors generated by the TensorFlow Lite model, based on
|
||||
# the SSD anchors and the specification in the options, into a vector of
|
||||
# detections. Each detection describes a detected object.
|
||||
node {
|
||||
calculator: "TfLiteTensorsToDetectionsCalculator"
|
||||
input_stream: "TENSORS_GPU:detection_tensors"
|
||||
input_side_packet: "ANCHORS:anchors"
|
||||
output_stream: "DETECTIONS:detections"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.TfLiteTensorsToDetectionsCalculatorOptions] {
|
||||
num_classes: 1
|
||||
num_boxes: 2944
|
||||
num_coords: 18
|
||||
box_coord_offset: 0
|
||||
keypoint_coord_offset: 4
|
||||
num_keypoints: 7
|
||||
num_values_per_keypoint: 2
|
||||
sigmoid_score: true
|
||||
score_clipping_thresh: 100.0
|
||||
reverse_output_order: true
|
||||
|
||||
x_scale: 256.0
|
||||
y_scale: 256.0
|
||||
h_scale: 256.0
|
||||
w_scale: 256.0
|
||||
min_score_thresh: 0.7
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Performs non-max suppression to remove excessive detections.
|
||||
node {
|
||||
calculator: "NonMaxSuppressionCalculator"
|
||||
input_stream: "detections"
|
||||
output_stream: "filtered_detections"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.NonMaxSuppressionCalculatorOptions] {
|
||||
min_suppression_threshold: 0.3
|
||||
overlap_type: INTERSECTION_OVER_UNION
|
||||
algorithm: WEIGHTED
|
||||
return_empty_detections: true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Maps detection label IDs to the corresponding label text ("Palm"). The label
|
||||
# map is provided in the label_map_path option.
|
||||
node {
|
||||
calculator: "DetectionLabelIdToTextCalculator"
|
||||
input_stream: "filtered_detections"
|
||||
output_stream: "labeled_detections"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.DetectionLabelIdToTextCalculatorOptions] {
|
||||
label_map_path: "mediapipe/models/palm_detection_labelmap.txt"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Adjusts detection locations (already normalized to [0.f, 1.f]) on the
|
||||
# letterboxed image (after image transformation with the FIT scale mode) to the
|
||||
# corresponding locations on the same image with the letterbox removed (the
|
||||
# input image to the graph before image transformation).
|
||||
node {
|
||||
calculator: "DetectionLetterboxRemovalCalculator"
|
||||
input_stream: "DETECTIONS:labeled_detections"
|
||||
input_stream: "LETTERBOX_PADDING:letterbox_padding"
|
||||
output_stream: "DETECTIONS:palm_detections"
|
||||
}
|
||||
|
||||
# Extracts image size from the input images.
|
||||
node {
|
||||
calculator: "ImagePropertiesCalculator"
|
||||
input_stream: "IMAGE_GPU:input_video"
|
||||
output_stream: "SIZE:image_size"
|
||||
}
|
||||
|
||||
# Converts each palm detection into a rectangle (normalized by image size)
|
||||
# that encloses the palm and is rotated such that the line connecting center of
|
||||
# the wrist and MCP of the middle finger is aligned with the Y-axis of the
|
||||
# rectangle.
|
||||
node {
|
||||
calculator: "DetectionsToRectsCalculator"
|
||||
input_stream: "DETECTIONS:palm_detections"
|
||||
input_stream: "IMAGE_SIZE:image_size"
|
||||
output_stream: "NORM_RECTS:palm_rects"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.DetectionsToRectsCalculatorOptions] {
|
||||
rotation_vector_start_keypoint_index: 0 # Center of wrist.
|
||||
rotation_vector_end_keypoint_index: 2 # MCP of middle finger.
|
||||
rotation_vector_target_angle_degrees: 90
|
||||
output_zero_rect_for_empty_detections: true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Expands and shifts the rectangle that contains the palm so that it's likely
|
||||
# to cover the entire hand.
|
||||
node {
|
||||
calculator: "RectTransformationCalculator"
|
||||
input_stream: "NORM_RECTS:palm_rects"
|
||||
input_stream: "IMAGE_SIZE:image_size"
|
||||
output_stream: "hand_rects_from_palm_detections"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.RectTransformationCalculatorOptions] {
|
||||
scale_x: 2.6
|
||||
scale_y: 2.6
|
||||
shift_y: -0.5
|
||||
square_long: true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Clips the size of the input vector to the provided max_vec_size. This
|
||||
# determines the maximum number of hand instances this graph outputs.
|
||||
# Note that the performance gain of clipping detections earlier in this graph is
|
||||
# minimal because NMS will minimize overlapping detections and the number of
|
||||
# detections isn't expected to exceed 5-10.
|
||||
node {
|
||||
calculator: "ClipNormalizedRectVectorSizeCalculator"
|
||||
input_stream: "hand_rects_from_palm_detections"
|
||||
output_stream: "clipped_hand_rects_from_palm_detections"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.ClipVectorSizeCalculatorOptions] {
|
||||
# This value can be changed to support tracking arbitrary number of hands.
|
||||
# Please also remember to modify min_size in
|
||||
# CollectionHsMinSizeCalculatorOptions in
|
||||
# mediapipe/graphs/hand_tracking/multi_hand_tracking_mobile.pbtxt and
|
||||
# mediapipe/graphs/hand_tracking/multi_hand_tracking_desktop_live.pbtxt.
|
||||
max_vec_size: 2
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,84 +0,0 @@
|
||||
# MediaPipe hand landmark localization subgraph.
|
||||
|
||||
type: "MultiHandLandmarkSubgraph"
|
||||
|
||||
input_stream: "IMAGE:input_video"
|
||||
# A vector of NormalizedRect, one per each hand detected.
|
||||
input_stream: "NORM_RECTS:multi_hand_rects"
|
||||
# A vector of NormalizedLandmarks, one set per each hand.
|
||||
output_stream: "LANDMARKS:filtered_multi_hand_landmarks"
|
||||
# A vector of NormalizedRect, one per each hand.
|
||||
output_stream: "NORM_RECTS:filtered_multi_hand_rects_for_next_frame"
|
||||
|
||||
# Outputs each element of multi_hand_rects at a fake timestamp for the rest
|
||||
# of the graph to process. Clones the input_video packet for each
|
||||
# single_hand_rect at the fake timestamp. At the end of the loop,
|
||||
# outputs the BATCH_END timestamp for downstream calculators to inform them
|
||||
# that all elements in the vector have been processed.
|
||||
node {
|
||||
calculator: "BeginLoopNormalizedRectCalculator"
|
||||
input_stream: "ITERABLE:multi_hand_rects"
|
||||
input_stream: "CLONE:input_video"
|
||||
output_stream: "ITEM:single_hand_rect"
|
||||
output_stream: "CLONE:input_video_cloned"
|
||||
output_stream: "BATCH_END:single_hand_rect_timestamp"
|
||||
}
|
||||
|
||||
node {
|
||||
calculator: "HandLandmarkSubgraph"
|
||||
input_stream: "IMAGE:input_video_cloned"
|
||||
input_stream: "NORM_RECT:single_hand_rect"
|
||||
output_stream: "LANDMARKS:single_hand_landmarks"
|
||||
output_stream: "NORM_RECT:single_hand_rect_from_landmarks"
|
||||
output_stream: "PRESENCE:single_hand_presence"
|
||||
}
|
||||
|
||||
# Collects the boolean presence value for each single hand into a vector. Upon
|
||||
# receiving the BATCH_END timestamp, outputs a vector of boolean values at the
|
||||
# BATCH_END timestamp.
|
||||
node {
|
||||
calculator: "EndLoopBooleanCalculator"
|
||||
input_stream: "ITEM:single_hand_presence"
|
||||
input_stream: "BATCH_END:single_hand_rect_timestamp"
|
||||
output_stream: "ITERABLE:multi_hand_presence"
|
||||
}
|
||||
|
||||
# Collects a set of landmarks for each hand into a vector. Upon receiving the
|
||||
# BATCH_END timestamp, outputs the vector of landmarks at the BATCH_END
|
||||
# timestamp.
|
||||
node {
|
||||
calculator: "EndLoopNormalizedLandmarkListVectorCalculator"
|
||||
input_stream: "ITEM:single_hand_landmarks"
|
||||
input_stream: "BATCH_END:single_hand_rect_timestamp"
|
||||
output_stream: "ITERABLE:multi_hand_landmarks"
|
||||
}
|
||||
|
||||
# Collects a NormalizedRect for each hand into a vector. Upon receiving the
|
||||
# BATCH_END timestamp, outputs the vector of NormalizedRect at the BATCH_END
|
||||
# timestamp.
|
||||
node {
|
||||
calculator: "EndLoopNormalizedRectCalculator"
|
||||
input_stream: "ITEM:single_hand_rect_from_landmarks"
|
||||
input_stream: "BATCH_END:single_hand_rect_timestamp"
|
||||
output_stream: "ITERABLE:multi_hand_rects_for_next_frame"
|
||||
}
|
||||
|
||||
# Filters the input vector of landmarks based on hand presence value for each
|
||||
# hand. If the hand presence for hand #i is false, the set of landmarks
|
||||
# corresponding to that hand are dropped from the vector.
|
||||
node {
|
||||
calculator: "FilterLandmarkListCollectionCalculator"
|
||||
input_stream: "ITERABLE:multi_hand_landmarks"
|
||||
input_stream: "CONDITION:multi_hand_presence"
|
||||
output_stream: "ITERABLE:filtered_multi_hand_landmarks"
|
||||
}
|
||||
|
||||
# Filters the input vector of NormalizedRect based on hand presence value for
|
||||
# each hand. If the hand presence for hand #i is false, the NormalizedRect
|
||||
# corresponding to that hand are dropped from the vector.
|
||||
node {
|
||||
calculator: "FilterNormalizedRectCollectionCalculator"
|
||||
input_stream: "ITERABLE:multi_hand_rects_for_next_frame"
|
||||
input_stream: "CONDITION:multi_hand_presence"
|
||||
output_stream: "ITERABLE:filtered_multi_hand_rects_for_next_frame"
|
||||
}
|
||||
@@ -1,123 +0,0 @@
|
||||
# MediaPipe hand tracking rendering subgraph.
|
||||
|
||||
type: "RendererSubgraph"
|
||||
|
||||
input_stream: "IMAGE:input_image"
|
||||
input_stream: "DETECTIONS:detections"
|
||||
input_stream: "LANDMARKS:landmarks"
|
||||
input_stream: "NORM_RECT:rect"
|
||||
input_stream: "HANDEDNESS:handedness"
|
||||
output_stream: "IMAGE:output_image"
|
||||
|
||||
# Converts classification to drawing primitives for annotation overlay.
|
||||
node {
|
||||
calculator: "LabelsToRenderDataCalculator"
|
||||
input_stream: "CLASSIFICATIONS:handedness"
|
||||
output_stream: "RENDER_DATA:handedness_render_data"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.LabelsToRenderDataCalculatorOptions]: {
|
||||
color { r: 255 g: 0 b: 0 }
|
||||
thickness: 10.0
|
||||
font_height_px: 50
|
||||
horizontal_offset_px: 100
|
||||
vertical_offset_px: 100
|
||||
|
||||
max_num_labels: 1
|
||||
location: TOP_LEFT
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Converts detections to drawing primitives for annotation overlay.
|
||||
node {
|
||||
calculator: "DetectionsToRenderDataCalculator"
|
||||
input_stream: "DETECTIONS:detections"
|
||||
output_stream: "RENDER_DATA:detection_render_data"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.DetectionsToRenderDataCalculatorOptions] {
|
||||
thickness: 4.0
|
||||
color { r: 0 g: 255 b: 0 }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Converts landmarks to drawing primitives for annotation overlay.
|
||||
node {
|
||||
calculator: "LandmarksToRenderDataCalculator"
|
||||
input_stream: "NORM_LANDMARKS:landmarks"
|
||||
output_stream: "RENDER_DATA:landmark_render_data"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.LandmarksToRenderDataCalculatorOptions] {
|
||||
landmark_connections: 0
|
||||
landmark_connections: 1
|
||||
landmark_connections: 1
|
||||
landmark_connections: 2
|
||||
landmark_connections: 2
|
||||
landmark_connections: 3
|
||||
landmark_connections: 3
|
||||
landmark_connections: 4
|
||||
landmark_connections: 0
|
||||
landmark_connections: 5
|
||||
landmark_connections: 5
|
||||
landmark_connections: 6
|
||||
landmark_connections: 6
|
||||
landmark_connections: 7
|
||||
landmark_connections: 7
|
||||
landmark_connections: 8
|
||||
landmark_connections: 5
|
||||
landmark_connections: 9
|
||||
landmark_connections: 9
|
||||
landmark_connections: 10
|
||||
landmark_connections: 10
|
||||
landmark_connections: 11
|
||||
landmark_connections: 11
|
||||
landmark_connections: 12
|
||||
landmark_connections: 9
|
||||
landmark_connections: 13
|
||||
landmark_connections: 13
|
||||
landmark_connections: 14
|
||||
landmark_connections: 14
|
||||
landmark_connections: 15
|
||||
landmark_connections: 15
|
||||
landmark_connections: 16
|
||||
landmark_connections: 13
|
||||
landmark_connections: 17
|
||||
landmark_connections: 0
|
||||
landmark_connections: 17
|
||||
landmark_connections: 17
|
||||
landmark_connections: 18
|
||||
landmark_connections: 18
|
||||
landmark_connections: 19
|
||||
landmark_connections: 19
|
||||
landmark_connections: 20
|
||||
landmark_color { r: 255 g: 0 b: 0 }
|
||||
connection_color { r: 0 g: 255 b: 0 }
|
||||
thickness: 4.0
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Converts normalized rects to drawing primitives for annotation overlay.
|
||||
node {
|
||||
calculator: "RectToRenderDataCalculator"
|
||||
input_stream: "NORM_RECT:rect"
|
||||
output_stream: "RENDER_DATA:rect_render_data"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.RectToRenderDataCalculatorOptions] {
|
||||
filled: false
|
||||
color { r: 255 g: 0 b: 0 }
|
||||
thickness: 4.0
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Draws annotations and overlays them on top of the input images.
|
||||
node {
|
||||
calculator: "AnnotationOverlayCalculator"
|
||||
input_stream: "IMAGE:input_image"
|
||||
input_stream: "detection_render_data"
|
||||
input_stream: "landmark_render_data"
|
||||
input_stream: "handedness_render_data"
|
||||
input_stream: "rect_render_data"
|
||||
output_stream: "IMAGE:output_image"
|
||||
}
|
||||
@@ -1,123 +0,0 @@
|
||||
# MediaPipe hand tracking rendering subgraph.
|
||||
|
||||
type: "RendererSubgraph"
|
||||
|
||||
input_stream: "IMAGE:input_image"
|
||||
input_stream: "DETECTIONS:detections"
|
||||
input_stream: "LANDMARKS:landmarks"
|
||||
input_stream: "NORM_RECT:rect"
|
||||
input_stream: "HANDEDNESS:handedness"
|
||||
output_stream: "IMAGE:output_image"
|
||||
|
||||
# Converts classification to drawing primitives for annotation overlay.
|
||||
node {
|
||||
calculator: "LabelsToRenderDataCalculator"
|
||||
input_stream: "CLASSIFICATIONS:handedness"
|
||||
output_stream: "RENDER_DATA:handedness_render_data"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.LabelsToRenderDataCalculatorOptions]: {
|
||||
color { r: 255 g: 0 b: 0 }
|
||||
thickness: 10.0
|
||||
font_height_px: 50
|
||||
horizontal_offset_px: 200
|
||||
vertical_offset_px: 100
|
||||
|
||||
max_num_labels: 1
|
||||
location: TOP_LEFT
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Converts detections to drawing primitives for annotation overlay.
|
||||
node {
|
||||
calculator: "DetectionsToRenderDataCalculator"
|
||||
input_stream: "DETECTIONS:detections"
|
||||
output_stream: "RENDER_DATA:detection_render_data"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.DetectionsToRenderDataCalculatorOptions] {
|
||||
thickness: 4.0
|
||||
color { r: 0 g: 255 b: 0 }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Converts landmarks to drawing primitives for annotation overlay.
|
||||
node {
|
||||
calculator: "LandmarksToRenderDataCalculator"
|
||||
input_stream: "NORM_LANDMARKS:landmarks"
|
||||
output_stream: "RENDER_DATA:landmark_render_data"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.LandmarksToRenderDataCalculatorOptions] {
|
||||
landmark_connections: 0
|
||||
landmark_connections: 1
|
||||
landmark_connections: 1
|
||||
landmark_connections: 2
|
||||
landmark_connections: 2
|
||||
landmark_connections: 3
|
||||
landmark_connections: 3
|
||||
landmark_connections: 4
|
||||
landmark_connections: 0
|
||||
landmark_connections: 5
|
||||
landmark_connections: 5
|
||||
landmark_connections: 6
|
||||
landmark_connections: 6
|
||||
landmark_connections: 7
|
||||
landmark_connections: 7
|
||||
landmark_connections: 8
|
||||
landmark_connections: 5
|
||||
landmark_connections: 9
|
||||
landmark_connections: 9
|
||||
landmark_connections: 10
|
||||
landmark_connections: 10
|
||||
landmark_connections: 11
|
||||
landmark_connections: 11
|
||||
landmark_connections: 12
|
||||
landmark_connections: 9
|
||||
landmark_connections: 13
|
||||
landmark_connections: 13
|
||||
landmark_connections: 14
|
||||
landmark_connections: 14
|
||||
landmark_connections: 15
|
||||
landmark_connections: 15
|
||||
landmark_connections: 16
|
||||
landmark_connections: 13
|
||||
landmark_connections: 17
|
||||
landmark_connections: 0
|
||||
landmark_connections: 17
|
||||
landmark_connections: 17
|
||||
landmark_connections: 18
|
||||
landmark_connections: 18
|
||||
landmark_connections: 19
|
||||
landmark_connections: 19
|
||||
landmark_connections: 20
|
||||
landmark_color { r: 255 g: 0 b: 0 }
|
||||
connection_color { r: 0 g: 255 b: 0 }
|
||||
thickness: 4.0
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Converts normalized rects to drawing primitives for annotation overlay.
|
||||
node {
|
||||
calculator: "RectToRenderDataCalculator"
|
||||
input_stream: "NORM_RECT:rect"
|
||||
output_stream: "RENDER_DATA:rect_render_data"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.RectToRenderDataCalculatorOptions] {
|
||||
filled: false
|
||||
color { r: 255 g: 0 b: 0 }
|
||||
thickness: 4.0
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Draws annotations and overlays them on top of the input images.
|
||||
node {
|
||||
calculator: "AnnotationOverlayCalculator"
|
||||
input_stream: "IMAGE_GPU:input_image"
|
||||
input_stream: "detection_render_data"
|
||||
input_stream: "landmark_render_data"
|
||||
input_stream: "handedness_render_data"
|
||||
input_stream: "rect_render_data"
|
||||
output_stream: "IMAGE_GPU:output_image"
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
# Copyright 2019 The MediaPipe Authors.
|
||||
# Copyright 2020 The MediaPipe Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
@@ -28,6 +28,23 @@ exports_files(glob([
|
||||
cc_library(
|
||||
name = "mobile_calculators",
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//mediapipe/calculators/core:constant_side_packet_calculator",
|
||||
"//mediapipe/calculators/core:flow_limiter_calculator",
|
||||
"//mediapipe/calculators/core:gate_calculator",
|
||||
"//mediapipe/calculators/core:merge_calculator",
|
||||
"//mediapipe/calculators/core:previous_loopback_calculator",
|
||||
"//mediapipe/calculators/image:image_cropping_calculator",
|
||||
"//mediapipe/graphs/object_detection_3d/calculators:annotations_to_model_matrices_calculator",
|
||||
"//mediapipe/graphs/object_detection_3d/calculators:gl_animation_overlay_calculator",
|
||||
"//mediapipe/graphs/object_detection_3d/subgraphs:box_landmark_gpu",
|
||||
"//mediapipe/graphs/object_detection_3d/subgraphs:object_detection_oid_v4_gpu",
|
||||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "mobile_calculators_1stage",
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//mediapipe/calculators/core:packet_resampler_calculator",
|
||||
"//mediapipe/calculators/image:image_cropping_calculator",
|
||||
@@ -40,17 +57,17 @@ cc_library(
|
||||
)
|
||||
|
||||
mediapipe_binary_graph(
|
||||
name = "mobile_gpu_binary_graph_shoe",
|
||||
graph = "shoe_classic_occlusion_tracking.pbtxt",
|
||||
output_name = "mobile_gpu_shoe.binarypb",
|
||||
name = "mobile_gpu_binary_graph",
|
||||
graph = "object_occlusion_tracking.pbtxt",
|
||||
output_name = "mobile_gpu_binary_graph.binarypb",
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":mobile_calculators"],
|
||||
)
|
||||
|
||||
mediapipe_binary_graph(
|
||||
name = "mobile_gpu_binary_graph_chair",
|
||||
graph = "chair_classic_occlusion_tracking.pbtxt",
|
||||
output_name = "mobile_gpu_chair.binarypb",
|
||||
name = "mobile_gpu_1stage_binary_graph",
|
||||
graph = "object_occlusion_tracking_1stage.pbtxt",
|
||||
output_name = "mobile_gpu_1stage_binary_graph.binarypb",
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":mobile_calculators"],
|
||||
deps = [":mobile_calculators_1stage"],
|
||||
)
|
||||
|
||||
@@ -12,67 +12,65 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
|
||||
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library")
|
||||
|
||||
licenses(["notice"])
|
||||
|
||||
package(default_visibility = ["//visibility:public"])
|
||||
|
||||
proto_library(
|
||||
mediapipe_proto_library(
|
||||
name = "object_proto",
|
||||
srcs = [
|
||||
"object.proto",
|
||||
],
|
||||
srcs = ["object.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
|
||||
proto_library(
|
||||
mediapipe_proto_library(
|
||||
name = "a_r_capture_metadata_proto",
|
||||
srcs = [
|
||||
"a_r_capture_metadata.proto",
|
||||
],
|
||||
srcs = ["a_r_capture_metadata.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
|
||||
proto_library(
|
||||
mediapipe_proto_library(
|
||||
name = "annotation_proto",
|
||||
srcs = [
|
||||
"annotation_data.proto",
|
||||
],
|
||||
srcs = ["annotation_data.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
":a_r_capture_metadata_proto",
|
||||
":object_proto",
|
||||
],
|
||||
)
|
||||
|
||||
proto_library(
|
||||
name = "belief_decoder_config_proto",
|
||||
srcs = [
|
||||
"belief_decoder_config.proto",
|
||||
],
|
||||
)
|
||||
|
||||
proto_library(
|
||||
mediapipe_proto_library(
|
||||
name = "camera_parameters_proto",
|
||||
srcs = [
|
||||
"camera_parameters.proto",
|
||||
],
|
||||
srcs = ["camera_parameters.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
|
||||
proto_library(
|
||||
mediapipe_proto_library(
|
||||
name = "frame_annotation_tracker_calculator_proto",
|
||||
srcs = ["frame_annotation_tracker_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_proto",
|
||||
],
|
||||
)
|
||||
|
||||
proto_library(
|
||||
mediapipe_proto_library(
|
||||
name = "gl_animation_overlay_calculator_proto",
|
||||
srcs = ["gl_animation_overlay_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = ["//mediapipe/framework:calculator_proto"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_proto",
|
||||
],
|
||||
)
|
||||
|
||||
proto_library(
|
||||
mediapipe_proto_library(
|
||||
name = "belief_decoder_config_proto",
|
||||
srcs = ["belief_decoder_config.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
|
||||
mediapipe_proto_library(
|
||||
name = "tflite_tensors_to_objects_calculator_proto",
|
||||
srcs = ["tflite_tensors_to_objects_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
@@ -82,7 +80,7 @@ proto_library(
|
||||
],
|
||||
)
|
||||
|
||||
proto_library(
|
||||
mediapipe_proto_library(
|
||||
name = "lift_2d_frame_annotation_to_3d_calculator_proto",
|
||||
srcs = ["lift_2d_frame_annotation_to_3d_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
@@ -92,7 +90,7 @@ proto_library(
|
||||
],
|
||||
)
|
||||
|
||||
proto_library(
|
||||
mediapipe_proto_library(
|
||||
name = "annotations_to_model_matrices_calculator_proto",
|
||||
srcs = ["annotations_to_model_matrices_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
@@ -101,7 +99,7 @@ proto_library(
|
||||
],
|
||||
)
|
||||
|
||||
proto_library(
|
||||
mediapipe_proto_library(
|
||||
name = "model_matrix_proto",
|
||||
srcs = ["model_matrix.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
@@ -110,7 +108,7 @@ proto_library(
|
||||
],
|
||||
)
|
||||
|
||||
proto_library(
|
||||
mediapipe_proto_library(
|
||||
name = "annotations_to_render_data_calculator_proto",
|
||||
srcs = ["annotations_to_render_data_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
@@ -120,112 +118,22 @@ proto_library(
|
||||
],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "object_cc_proto",
|
||||
srcs = ["object.proto"],
|
||||
mediapipe_proto_library(
|
||||
name = "frame_annotation_to_rect_calculator_proto",
|
||||
srcs = ["frame_annotation_to_rect_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":object_proto"],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "a_r_capture_metadata_cc_proto",
|
||||
srcs = ["a_r_capture_metadata.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":a_r_capture_metadata_proto"],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "annotation_cc_proto",
|
||||
srcs = ["annotation_data.proto"],
|
||||
cc_deps = [
|
||||
":a_r_capture_metadata_cc_proto",
|
||||
":object_cc_proto",
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_proto",
|
||||
],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":annotation_proto"],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "camera_parameters_cc_proto",
|
||||
srcs = ["camera_parameters.proto"],
|
||||
mediapipe_proto_library(
|
||||
name = "filter_detection_calculator_proto",
|
||||
srcs = ["filter_detection_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":camera_parameters_proto"],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "frame_annotation_tracker_calculator_cc_proto",
|
||||
srcs = ["frame_annotation_tracker_calculator.proto"],
|
||||
cc_deps = [
|
||||
"//mediapipe/framework:calculator_cc_proto",
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_proto",
|
||||
],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":frame_annotation_tracker_calculator_proto"],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "gl_animation_overlay_calculator_cc_proto",
|
||||
srcs = ["gl_animation_overlay_calculator.proto"],
|
||||
cc_deps = [
|
||||
"//mediapipe/framework:calculator_cc_proto",
|
||||
],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":gl_animation_overlay_calculator_proto"],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "belief_decoder_config_cc_proto",
|
||||
srcs = ["belief_decoder_config.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":belief_decoder_config_proto"],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "tflite_tensors_to_objects_calculator_cc_proto",
|
||||
srcs = ["tflite_tensors_to_objects_calculator.proto"],
|
||||
cc_deps = [
|
||||
":belief_decoder_config_cc_proto",
|
||||
"//mediapipe/framework:calculator_cc_proto",
|
||||
],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":tflite_tensors_to_objects_calculator_proto"],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "lift_2d_frame_annotation_to_3d_calculator_cc_proto",
|
||||
srcs = ["lift_2d_frame_annotation_to_3d_calculator.proto"],
|
||||
cc_deps = [
|
||||
":belief_decoder_config_cc_proto",
|
||||
"//mediapipe/framework:calculator_cc_proto",
|
||||
],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":lift_2d_frame_annotation_to_3d_calculator_proto"],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "annotations_to_model_matrices_calculator_cc_proto",
|
||||
srcs = ["annotations_to_model_matrices_calculator.proto"],
|
||||
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":annotations_to_model_matrices_calculator_proto"],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "model_matrix_cc_proto",
|
||||
srcs = ["model_matrix.proto"],
|
||||
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":model_matrix_proto"],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "annotations_to_render_data_calculator_cc_proto",
|
||||
srcs = ["annotations_to_render_data_calculator.proto"],
|
||||
cc_deps = [
|
||||
"//mediapipe/framework:calculator_cc_proto",
|
||||
"//mediapipe/util:color_cc_proto",
|
||||
],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":annotations_to_render_data_calculator_proto"],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
@@ -452,6 +360,55 @@ cc_library(
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "frame_annotation_to_rect_calculator",
|
||||
srcs = ["frame_annotation_to_rect_calculator.cc"],
|
||||
deps = [
|
||||
":annotation_cc_proto",
|
||||
":box",
|
||||
":frame_annotation_to_rect_calculator_cc_proto",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/formats:rect_cc_proto",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/port:status",
|
||||
"@com_google_absl//absl/memory",
|
||||
"@eigen_archive//:eigen",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "landmarks_to_frame_annotation_calculator",
|
||||
srcs = ["landmarks_to_frame_annotation_calculator.cc"],
|
||||
deps = [
|
||||
":annotation_cc_proto",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/formats:landmark_cc_proto",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/port:status",
|
||||
"@com_google_absl//absl/memory",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "filter_detection_calculator",
|
||||
srcs = ["filter_detection_calculator.cc"],
|
||||
deps = [
|
||||
":filter_detection_calculator_cc_proto",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/formats:detection_cc_proto",
|
||||
"//mediapipe/framework/formats:location_data_cc_proto",
|
||||
"//mediapipe/framework/port:logging",
|
||||
"//mediapipe/framework/port:map_util",
|
||||
"//mediapipe/framework/port:re2",
|
||||
"//mediapipe/framework/port:status",
|
||||
"@com_google_absl//absl/container:node_hash_set",
|
||||
"@com_google_absl//absl/strings",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_test(
|
||||
name = "box_util_test",
|
||||
srcs = ["box_util_test.cc"],
|
||||
|
||||
+16
-2
@@ -93,6 +93,14 @@ REGISTER_CALCULATOR(AnnotationsToModelMatricesCalculator);
|
||||
if (cc->Outputs().HasTag(kModelMatricesTag)) {
|
||||
cc->Outputs().Tag(kModelMatricesTag).Set<TimedModelMatrixProtoList>();
|
||||
}
|
||||
|
||||
if (cc->InputSidePackets().HasTag("MODEL_SCALE")) {
|
||||
cc->InputSidePackets().Tag("MODEL_SCALE").Set<float[]>();
|
||||
}
|
||||
|
||||
if (cc->InputSidePackets().HasTag("MODEL_TRANSFORMATION")) {
|
||||
cc->InputSidePackets().Tag("MODEL_TRANSFORMATION").Set<float[]>();
|
||||
}
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
@@ -103,14 +111,20 @@ REGISTER_CALCULATOR(AnnotationsToModelMatricesCalculator);
|
||||
cc->SetOffset(TimestampDiff(0));
|
||||
options_ = cc->Options<AnnotationsToModelMatricesCalculatorOptions>();
|
||||
|
||||
if (options_.model_scale_size() == 3) {
|
||||
if (cc->InputSidePackets().HasTag("MODEL_SCALE")) {
|
||||
model_scale_ = Eigen::Map<const Eigen::Vector3f>(
|
||||
cc->InputSidePackets().Tag("MODEL_SCALE").Get<float[]>());
|
||||
} else if (options_.model_scale_size() == 3) {
|
||||
model_scale_ =
|
||||
Eigen::Map<const Eigen::Vector3f>(options_.model_scale().data());
|
||||
} else {
|
||||
model_scale_.setOnes();
|
||||
}
|
||||
|
||||
if (options_.model_transformation_size() == 16) {
|
||||
if (cc->InputSidePackets().HasTag("MODEL_TRANSFORMATION")) {
|
||||
model_transformation_ = Eigen::Map<const Matrix4fRM>(
|
||||
cc->InputSidePackets().Tag("MODEL_TRANSFORMATION").Get<float[]>());
|
||||
} else if (options_.model_transformation_size() == 16) {
|
||||
model_transformation_ =
|
||||
Eigen::Map<const Matrix4fRM>(options_.model_transformation().data());
|
||||
} else {
|
||||
|
||||
@@ -0,0 +1,265 @@
|
||||
// Copyright 2020 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "absl/container/node_hash_set.h"
|
||||
#include "absl/strings/str_split.h"
|
||||
#include "absl/strings/string_view.h"
|
||||
#include "absl/strings/strip.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/formats/detection.pb.h"
|
||||
#include "mediapipe/framework/formats/location_data.pb.h"
|
||||
#include "mediapipe/framework/port/logging.h"
|
||||
#include "mediapipe/framework/port/map_util.h"
|
||||
#include "mediapipe/framework/port/re2.h"
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
#include "mediapipe/graphs/object_detection_3d/calculators/filter_detection_calculator.pb.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
namespace {
|
||||
|
||||
constexpr char kDetectionTag[] = "DETECTION";
|
||||
constexpr char kDetectionsTag[] = "DETECTIONS";
|
||||
constexpr char kLabelsTag[] = "LABELS";
|
||||
constexpr char kLabelsCsvTag[] = "LABELS_CSV";
|
||||
|
||||
using ::mediapipe::ContainsKey;
|
||||
using ::mediapipe::RE2;
|
||||
using Detections = std::vector<Detection>;
|
||||
using Strings = std::vector<std::string>;
|
||||
|
||||
} // namespace
|
||||
|
||||
// Filters the entries in a Detection to only those with valid scores
|
||||
// for the specified allowed labels. Allowed labels are provided as a
|
||||
// vector<std::string> in an optional input side packet. Allowed labels can
|
||||
// contain simple strings or regular expressions. The valid score range
|
||||
// can be set in the options.The allowed labels can be provided as
|
||||
// vector<std::string> (LABELS) or CSV std::string (LABELS_CSV) containing class
|
||||
// names of allowed labels. Note: Providing an empty vector in the input side
|
||||
// packet Packet causes this calculator to act as a sink if
|
||||
// empty_allowed_labels_means_allow_everything is set to false (default value).
|
||||
// To allow all labels, use the calculator with no input side packet stream, or
|
||||
// set empty_allowed_labels_means_allow_everything to true.
|
||||
//
|
||||
// Example config:
|
||||
// node {
|
||||
// calculator: "FilterDetectionCalculator"
|
||||
// input_stream: "DETECTIONS:detections"
|
||||
// output_stream: "DETECTIONS:filtered_detections"
|
||||
// input_side_packet: "LABELS:allowed_labels"
|
||||
// node_options: {
|
||||
// [type.googleapis.com/mediapipe.FilterDetectionCalculatorOptions]: {
|
||||
// min_score: 0.5
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
|
||||
struct FirstGreaterComparator {
|
||||
bool operator()(const std::pair<float, int>& a,
|
||||
const std::pair<float, int>& b) const {
|
||||
return a.first > b.first;
|
||||
}
|
||||
};
|
||||
|
||||
::mediapipe::Status SortLabelsByDecreasingScore(const Detection& detection,
|
||||
Detection* sorted_detection) {
|
||||
RET_CHECK(sorted_detection);
|
||||
RET_CHECK_EQ(detection.score_size(), detection.label_size());
|
||||
if (!detection.label_id().empty()) {
|
||||
RET_CHECK_EQ(detection.score_size(), detection.label_id_size());
|
||||
}
|
||||
// Copies input to keep all fields unchanged, and to reserve space for
|
||||
// repeated fields. Repeated fields (score, label, and label_id) will be
|
||||
// overwritten.
|
||||
*sorted_detection = detection;
|
||||
|
||||
std::vector<std::pair<float, int>> scores_and_indices(detection.score_size());
|
||||
for (int i = 0; i < detection.score_size(); ++i) {
|
||||
scores_and_indices[i].first = detection.score(i);
|
||||
scores_and_indices[i].second = i;
|
||||
}
|
||||
|
||||
std::sort(scores_and_indices.begin(), scores_and_indices.end(),
|
||||
FirstGreaterComparator());
|
||||
|
||||
for (int i = 0; i < detection.score_size(); ++i) {
|
||||
const int index = scores_and_indices[i].second;
|
||||
sorted_detection->set_score(i, detection.score(index));
|
||||
sorted_detection->set_label(i, detection.label(index));
|
||||
}
|
||||
|
||||
if (!detection.label_id().empty()) {
|
||||
for (int i = 0; i < detection.score_size(); ++i) {
|
||||
const int index = scores_and_indices[i].second;
|
||||
sorted_detection->set_label_id(i, detection.label_id(index));
|
||||
}
|
||||
}
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
class FilterDetectionCalculator : public CalculatorBase {
|
||||
public:
|
||||
static ::mediapipe::Status GetContract(CalculatorContract* cc);
|
||||
::mediapipe::Status Open(CalculatorContext* cc) override;
|
||||
::mediapipe::Status Process(CalculatorContext* cc) override;
|
||||
|
||||
private:
|
||||
bool IsValidLabel(const std::string& label);
|
||||
bool IsValidScore(float score);
|
||||
// Stores numeric limits for filtering on the score.
|
||||
FilterDetectionCalculatorOptions options_;
|
||||
// We use the next two fields to possibly filter to a limited set of
|
||||
// classes. The hash_set will be empty in two cases: 1) if no input
|
||||
// side packet stream is provided (not filtering on labels), or 2)
|
||||
// if the input side packet contains an empty vector (no labels are
|
||||
// allowed). We use limit_labels_ to distinguish between the two cases.
|
||||
bool limit_labels_ = true;
|
||||
absl::node_hash_set<std::string> allowed_labels_;
|
||||
};
|
||||
REGISTER_CALCULATOR(FilterDetectionCalculator);
|
||||
|
||||
::mediapipe::Status FilterDetectionCalculator::GetContract(
|
||||
CalculatorContract* cc) {
|
||||
RET_CHECK(!cc->Inputs().GetTags().empty());
|
||||
RET_CHECK(!cc->Outputs().GetTags().empty());
|
||||
|
||||
if (cc->Inputs().HasTag(kDetectionTag)) {
|
||||
cc->Inputs().Tag(kDetectionTag).Set<Detection>();
|
||||
cc->Outputs().Tag(kDetectionTag).Set<Detection>();
|
||||
}
|
||||
if (cc->Inputs().HasTag(kDetectionsTag)) {
|
||||
cc->Inputs().Tag(kDetectionsTag).Set<Detections>();
|
||||
cc->Outputs().Tag(kDetectionsTag).Set<Detections>();
|
||||
}
|
||||
if (cc->InputSidePackets().HasTag(kLabelsTag)) {
|
||||
cc->InputSidePackets().Tag(kLabelsTag).Set<Strings>();
|
||||
}
|
||||
if (cc->InputSidePackets().HasTag(kLabelsCsvTag)) {
|
||||
cc->InputSidePackets().Tag(kLabelsCsvTag).Set<std::string>();
|
||||
}
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status FilterDetectionCalculator::Open(CalculatorContext* cc) {
|
||||
cc->SetOffset(TimestampDiff(0));
|
||||
options_ = cc->Options<FilterDetectionCalculatorOptions>();
|
||||
limit_labels_ = cc->InputSidePackets().HasTag(kLabelsTag) ||
|
||||
cc->InputSidePackets().HasTag(kLabelsCsvTag);
|
||||
if (limit_labels_) {
|
||||
Strings whitelist_labels;
|
||||
if (cc->InputSidePackets().HasTag(kLabelsCsvTag)) {
|
||||
whitelist_labels = absl::StrSplit(
|
||||
cc->InputSidePackets().Tag(kLabelsCsvTag).Get<std::string>(), ',',
|
||||
absl::SkipWhitespace());
|
||||
for (auto& e : whitelist_labels) {
|
||||
absl::StripAsciiWhitespace(&e);
|
||||
}
|
||||
} else {
|
||||
whitelist_labels = cc->InputSidePackets().Tag(kLabelsTag).Get<Strings>();
|
||||
}
|
||||
allowed_labels_.insert(whitelist_labels.begin(), whitelist_labels.end());
|
||||
}
|
||||
if (limit_labels_ && allowed_labels_.empty()) {
|
||||
if (options_.fail_on_empty_labels()) {
|
||||
cc->GetCounter("VideosWithEmptyLabelsWhitelist")->Increment();
|
||||
return tool::StatusFail(
|
||||
"FilterDetectionCalculator received empty whitelist with "
|
||||
"fail_on_empty_labels = true.");
|
||||
}
|
||||
if (options_.empty_allowed_labels_means_allow_everything()) {
|
||||
// Continue as if side_input was not provided, i.e. pass all labels.
|
||||
limit_labels_ = false;
|
||||
}
|
||||
}
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status FilterDetectionCalculator::Process(CalculatorContext* cc) {
|
||||
if (limit_labels_ && allowed_labels_.empty()) {
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
Detections detections;
|
||||
if (cc->Inputs().HasTag(kDetectionsTag)) {
|
||||
detections = cc->Inputs().Tag(kDetectionsTag).Get<Detections>();
|
||||
} else if (cc->Inputs().HasTag(kDetectionTag)) {
|
||||
detections.emplace_back(cc->Inputs().Tag(kDetectionsTag).Get<Detection>());
|
||||
}
|
||||
std::unique_ptr<Detections> outputs(new Detections);
|
||||
for (const auto& input : detections) {
|
||||
Detection output;
|
||||
for (int i = 0; i < input.label_size(); ++i) {
|
||||
const std::string& label = input.label(i);
|
||||
const float score = input.score(i);
|
||||
if (IsValidLabel(label) && IsValidScore(score)) {
|
||||
output.add_label(label);
|
||||
output.add_score(score);
|
||||
}
|
||||
}
|
||||
if (output.label_size() > 0) {
|
||||
if (input.has_location_data()) {
|
||||
*output.mutable_location_data() = input.location_data();
|
||||
}
|
||||
Detection output_sorted;
|
||||
if (!SortLabelsByDecreasingScore(output, &output_sorted).ok()) {
|
||||
// Uses the orginal output if fails to sort.
|
||||
cc->GetCounter("FailedToSortLabelsInDetection")->Increment();
|
||||
output_sorted = output;
|
||||
}
|
||||
outputs->emplace_back(output_sorted);
|
||||
}
|
||||
}
|
||||
|
||||
if (cc->Outputs().HasTag(kDetectionsTag)) {
|
||||
cc->Outputs()
|
||||
.Tag(kDetectionsTag)
|
||||
.Add(outputs.release(), cc->InputTimestamp());
|
||||
} else if (!outputs->empty()) {
|
||||
cc->Outputs()
|
||||
.Tag(kDetectionsTag)
|
||||
.Add(new Detection((*outputs)[0]), cc->InputTimestamp());
|
||||
}
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
bool FilterDetectionCalculator::IsValidLabel(const std::string& label) {
|
||||
bool match = !limit_labels_ || ContainsKey(allowed_labels_, label);
|
||||
if (!match) {
|
||||
// If no exact match is found, check for regular expression
|
||||
// comparions in the allowed_labels.
|
||||
for (const auto& label_regexp : allowed_labels_) {
|
||||
match = match || RE2::FullMatch(label, RE2(label_regexp));
|
||||
}
|
||||
}
|
||||
return match;
|
||||
}
|
||||
|
||||
bool FilterDetectionCalculator::IsValidScore(float score) {
|
||||
if (options_.has_min_score() && score < options_.min_score()) {
|
||||
LOG(ERROR) << "Filter out detection with low score " << score;
|
||||
return false;
|
||||
}
|
||||
if (options_.has_max_score() && score > options_.max_score()) {
|
||||
LOG(ERROR) << "Filter out detection with high score " << score;
|
||||
return false;
|
||||
}
|
||||
LOG(ERROR) << "Pass detection with score " << score;
|
||||
return true;
|
||||
}
|
||||
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,45 @@
|
||||
// Copyright 2020 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
syntax = "proto2";
|
||||
|
||||
package mediapipe;
|
||||
|
||||
import "mediapipe/framework/calculator.proto";
|
||||
|
||||
message FilterDetectionCalculatorOptions {
|
||||
extend CalculatorOptions {
|
||||
optional FilterDetectionCalculatorOptions ext = 339582987;
|
||||
}
|
||||
optional float min_score = 1;
|
||||
optional float max_score = 2;
|
||||
// Setting fail_on_empty_labels to true will cause the calculator to return a
|
||||
// failure status on Open() if an empty list is provided on the external
|
||||
// input, immediately terminating the graph run.
|
||||
optional bool fail_on_empty_labels = 3 [default = false];
|
||||
// If fail_on_empty_labels is set to false setting
|
||||
// empty_allowed_labels_means_allow_everything to
|
||||
// false will cause the calculator to close output stream and ignore remaining
|
||||
// inputs if an empty list is provided. If
|
||||
// empty_allowed_labels_means_allow_everything is set to true this will force
|
||||
// calculator to pass all labels.
|
||||
optional bool empty_allowed_labels_means_allow_everything = 6
|
||||
[default = false];
|
||||
// Determines whether the input format is a vector<Detection> (use-case object
|
||||
// detectors) or Detection (use-case classifiers).
|
||||
optional bool use_detection_vector = 4 [deprecated = true];
|
||||
// Determines whether the input side packet format is a vector of labels, or
|
||||
// a string with comma separated labels.
|
||||
optional bool use_allowed_labels_csv = 5 [deprecated = true];
|
||||
}
|
||||
+185
@@ -0,0 +1,185 @@
|
||||
// Copyright 2020 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
|
||||
#include <cmath>
|
||||
|
||||
#include "Eigen/Dense"
|
||||
#include "absl/memory/memory.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/formats/rect.pb.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
#include "mediapipe/graphs/object_detection_3d/calculators/annotation_data.pb.h"
|
||||
#include "mediapipe/graphs/object_detection_3d/calculators/box.h"
|
||||
#include "mediapipe/graphs/object_detection_3d/calculators/frame_annotation_to_rect_calculator.pb.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
using Matrix3fRM = Eigen::Matrix<float, 3, 3, Eigen::RowMajor>;
|
||||
using Eigen::Vector2f;
|
||||
using Eigen::Vector3f;
|
||||
|
||||
namespace {
|
||||
|
||||
constexpr char kInputFrameAnnotationTag[] = "FRAME_ANNOTATION";
|
||||
constexpr char kOutputNormRectTag[] = "NORM_RECT";
|
||||
|
||||
} // namespace
|
||||
|
||||
// A calculator that converts FrameAnnotation proto to NormalizedRect.
|
||||
// The rotation angle of the NormalizedRect is derived from object's 3d pose.
|
||||
// The angle is calculated such that after rotation the 2d projection of y-axis.
|
||||
// on the image plane is always vertical.
|
||||
class FrameAnnotationToRectCalculator : public CalculatorBase {
|
||||
public:
|
||||
enum ViewStatus {
|
||||
TOP_VIEW_ON,
|
||||
TOP_VIEW_OFF,
|
||||
};
|
||||
|
||||
static ::mediapipe::Status GetContract(CalculatorContract* cc);
|
||||
::mediapipe::Status Open(CalculatorContext* cc) override;
|
||||
::mediapipe::Status Process(CalculatorContext* cc) override;
|
||||
|
||||
private:
|
||||
void AnnotationToRect(const FrameAnnotation& annotation,
|
||||
NormalizedRect* rect);
|
||||
float RotationAngleFromAnnotation(const FrameAnnotation& annotation);
|
||||
|
||||
float RotationAngleFromPose(const Matrix3fRM& rotation,
|
||||
const Vector3f& translation, const Vector3f& vec);
|
||||
ViewStatus status_;
|
||||
float off_threshold_;
|
||||
float on_threshold_;
|
||||
};
|
||||
REGISTER_CALCULATOR(FrameAnnotationToRectCalculator);
|
||||
|
||||
::mediapipe::Status FrameAnnotationToRectCalculator::Open(
|
||||
CalculatorContext* cc) {
|
||||
status_ = TOP_VIEW_OFF;
|
||||
const auto& options = cc->Options<FrameAnnotationToRectCalculatorOptions>();
|
||||
off_threshold_ = options.off_threshold();
|
||||
on_threshold_ = options.on_threshold();
|
||||
RET_CHECK(off_threshold_ <= on_threshold_);
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status FrameAnnotationToRectCalculator::GetContract(
|
||||
CalculatorContract* cc) {
|
||||
RET_CHECK(!cc->Inputs().GetTags().empty());
|
||||
RET_CHECK(!cc->Outputs().GetTags().empty());
|
||||
|
||||
if (cc->Inputs().HasTag(kInputFrameAnnotationTag)) {
|
||||
cc->Inputs().Tag(kInputFrameAnnotationTag).Set<FrameAnnotation>();
|
||||
}
|
||||
|
||||
if (cc->Outputs().HasTag(kOutputNormRectTag)) {
|
||||
cc->Outputs().Tag(kOutputNormRectTag).Set<NormalizedRect>();
|
||||
}
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status FrameAnnotationToRectCalculator::Process(
|
||||
CalculatorContext* cc) {
|
||||
if (cc->Inputs().Tag(kInputFrameAnnotationTag).IsEmpty()) {
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
auto output_rect = absl::make_unique<NormalizedRect>();
|
||||
AnnotationToRect(
|
||||
cc->Inputs().Tag(kInputFrameAnnotationTag).Get<FrameAnnotation>(),
|
||||
output_rect.get());
|
||||
|
||||
// Output
|
||||
cc->Outputs()
|
||||
.Tag(kOutputNormRectTag)
|
||||
.Add(output_rect.release(), cc->InputTimestamp());
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
void FrameAnnotationToRectCalculator::AnnotationToRect(
|
||||
const FrameAnnotation& annotation, NormalizedRect* rect) {
|
||||
float x_min = std::numeric_limits<float>::max();
|
||||
float x_max = std::numeric_limits<float>::min();
|
||||
float y_min = std::numeric_limits<float>::max();
|
||||
float y_max = std::numeric_limits<float>::min();
|
||||
const auto& object = annotation.annotations(0);
|
||||
for (const auto& keypoint : object.keypoints()) {
|
||||
const auto& point_2d = keypoint.point_2d();
|
||||
x_min = std::min(x_min, point_2d.x());
|
||||
x_max = std::max(x_max, point_2d.x());
|
||||
y_min = std::min(y_min, point_2d.y());
|
||||
y_max = std::max(y_max, point_2d.y());
|
||||
}
|
||||
rect->set_x_center((x_min + x_max) / 2);
|
||||
rect->set_y_center((y_min + y_max) / 2);
|
||||
rect->set_width(x_max - x_min);
|
||||
rect->set_height(y_max - y_min);
|
||||
rect->set_rotation(RotationAngleFromAnnotation(annotation));
|
||||
}
|
||||
|
||||
float FrameAnnotationToRectCalculator::RotationAngleFromAnnotation(
|
||||
const FrameAnnotation& annotation) {
|
||||
const auto& object = annotation.annotations(0);
|
||||
Box box("category");
|
||||
std::vector<Vector3f> vertices_3d;
|
||||
std::vector<Vector2f> vertices_2d;
|
||||
for (const auto& keypoint : object.keypoints()) {
|
||||
const auto& point_3d = keypoint.point_3d();
|
||||
const auto& point_2d = keypoint.point_2d();
|
||||
vertices_3d.emplace_back(
|
||||
Vector3f(point_3d.x(), point_3d.y(), point_3d.z()));
|
||||
vertices_2d.emplace_back(Vector2f(point_2d.x(), point_2d.y()));
|
||||
}
|
||||
box.Fit(vertices_3d);
|
||||
Vector3f scale = box.GetScale();
|
||||
Matrix3fRM box_rotation = box.GetRotation();
|
||||
Vector3f box_translation = box.GetTranslation();
|
||||
|
||||
// Rotation angle to use when top-view is on(top-view on),
|
||||
// Which will make z-axis upright after the rotation.
|
||||
const float angle_on =
|
||||
RotationAngleFromPose(box_rotation, box_translation, Vector3f::UnitZ());
|
||||
// Rotation angle to use when side-view is on(top-view off),
|
||||
// Which will make y-axis upright after the rotation.
|
||||
const float angle_off =
|
||||
RotationAngleFromPose(box_rotation, box_translation, Vector3f::UnitY());
|
||||
|
||||
// Calculate angle between z-axis and viewing ray in degrees.
|
||||
const float view_to_z_angle = std::acos(box_rotation(2, 1)) * 180 / M_PI;
|
||||
|
||||
// Determine threshold based on current status,
|
||||
// on_threshold_ is used for TOP_VIEW_ON -> TOP_VIEW_OFF transition,
|
||||
// off_threshold_ is used for TOP_VIEW_OFF -> TOP_VIEW_ON transition.
|
||||
const float thresh =
|
||||
(status_ == TOP_VIEW_ON) ? on_threshold_ : off_threshold_;
|
||||
|
||||
// If view_to_z_angle is smaller than threshold, then top-view is on;
|
||||
// Otherwise top-view is off.
|
||||
status_ = (view_to_z_angle < thresh) ? TOP_VIEW_ON : TOP_VIEW_OFF;
|
||||
|
||||
// Determine which angle to used based on current status_.
|
||||
float angle_to_rotate = (status_ == TOP_VIEW_ON) ? angle_on : angle_off;
|
||||
return angle_to_rotate;
|
||||
}
|
||||
|
||||
float FrameAnnotationToRectCalculator::RotationAngleFromPose(
|
||||
const Matrix3fRM& rotation, const Vector3f& translation,
|
||||
const Vector3f& vec) {
|
||||
auto p1 = rotation * vec + translation;
|
||||
auto p2 = -rotation * vec + translation;
|
||||
const float dy = p2[2] * p1[1] - p1[2] * p2[1];
|
||||
const float dx = p2[2] * p1[0] - p1[2] * p2[0];
|
||||
return std::atan2(-dy, dx);
|
||||
}
|
||||
|
||||
} // namespace mediapipe
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
// Copyright 2020 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
syntax = "proto2";
|
||||
|
||||
package mediapipe;
|
||||
|
||||
import "mediapipe/framework/calculator.proto";
|
||||
|
||||
message FrameAnnotationToRectCalculatorOptions {
|
||||
extend CalculatorOptions {
|
||||
optional FrameAnnotationToRectCalculatorOptions ext = 338119067;
|
||||
}
|
||||
|
||||
// The threshold to use when top-view is off,to enable hysteresis,
|
||||
// It's required that off_threshold <= on_threshold.
|
||||
optional float off_threshold = 1 [default = 40.0];
|
||||
// The threshold to use when top-view is on.
|
||||
optional float on_threshold = 2 [default = 41.0];
|
||||
}
|
||||
+76
@@ -0,0 +1,76 @@
|
||||
// Copyright 2020 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
|
||||
#include "absl/memory/memory.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/formats/landmark.pb.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
#include "mediapipe/graphs/object_detection_3d/calculators/annotation_data.pb.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
namespace {
|
||||
|
||||
constexpr char kInputLandmarksTag[] = "LANDMARKS";
|
||||
constexpr char kOutputFrameAnnotationTag[] = "FRAME_ANNOTATION";
|
||||
|
||||
} // namespace
|
||||
|
||||
// A calculator that converts NormalizedLandmarkList to FrameAnnotation proto.
|
||||
class LandmarksToFrameAnnotationCalculator : public CalculatorBase {
|
||||
public:
|
||||
static ::mediapipe::Status GetContract(CalculatorContract* cc);
|
||||
::mediapipe::Status Process(CalculatorContext* cc) override;
|
||||
};
|
||||
REGISTER_CALCULATOR(LandmarksToFrameAnnotationCalculator);
|
||||
|
||||
::mediapipe::Status LandmarksToFrameAnnotationCalculator::GetContract(
|
||||
CalculatorContract* cc) {
|
||||
RET_CHECK(!cc->Inputs().GetTags().empty());
|
||||
RET_CHECK(!cc->Outputs().GetTags().empty());
|
||||
|
||||
if (cc->Inputs().HasTag(kInputLandmarksTag)) {
|
||||
cc->Inputs().Tag(kInputLandmarksTag).Set<NormalizedLandmarkList>();
|
||||
}
|
||||
|
||||
if (cc->Outputs().HasTag(kOutputFrameAnnotationTag)) {
|
||||
cc->Outputs().Tag(kOutputFrameAnnotationTag).Set<FrameAnnotation>();
|
||||
}
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status LandmarksToFrameAnnotationCalculator::Process(
|
||||
CalculatorContext* cc) {
|
||||
auto frame_annotation = absl::make_unique<FrameAnnotation>();
|
||||
auto* box_annotation = frame_annotation->add_annotations();
|
||||
|
||||
const auto& landmarks =
|
||||
cc->Inputs().Tag(kInputLandmarksTag).Get<NormalizedLandmarkList>();
|
||||
RET_CHECK_GT(landmarks.landmark_size(), 0)
|
||||
<< "Input landmark vector is empty.";
|
||||
for (int i = 0; i < landmarks.landmark_size(); ++i) {
|
||||
auto* point2d = box_annotation->add_keypoints()->mutable_point_2d();
|
||||
point2d->set_x(landmarks.landmark(i).x());
|
||||
point2d->set_y(landmarks.landmark(i).y());
|
||||
}
|
||||
// Output
|
||||
if (cc->Outputs().HasTag(kOutputFrameAnnotationTag)) {
|
||||
cc->Outputs()
|
||||
.Tag(kOutputFrameAnnotationTag)
|
||||
.Add(frame_annotation.release(), cc->InputTimestamp());
|
||||
}
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,189 @@
|
||||
# MediaPipe graph that performs box tracking with TensorFlow Lite on GPU.
|
||||
|
||||
# Images coming into and out of the graph.
|
||||
input_stream: "input_video"
|
||||
input_stream: "input_width"
|
||||
input_stream: "input_height"
|
||||
output_stream: "output_video"
|
||||
|
||||
# Throttles the images flowing downstream for flow control. It passes through
|
||||
# the very first incoming image unaltered, and waits for downstream nodes
|
||||
# (calculators and subgraphs) in the graph to finish their tasks before it
|
||||
# passes through another image. All images that come in while waiting are
|
||||
# dropped, limiting the number of in-flight images in most part of the graph to
|
||||
# 1. This prevents the downstream nodes from queuing up incoming images and data
|
||||
# excessively, which leads to increased latency and memory usage, unwanted in
|
||||
# real-time mobile applications. It also eliminates unnecessarily computation,
|
||||
# e.g., the output produced by a node may get dropped downstream if the
|
||||
# subsequent nodes are still busy processing previous inputs.
|
||||
node {
|
||||
calculator: "FlowLimiterCalculator"
|
||||
input_stream: "input_video"
|
||||
input_stream: "FINISHED:box_rect"
|
||||
input_stream_info: {
|
||||
tag_index: "FINISHED"
|
||||
back_edge: true
|
||||
}
|
||||
output_stream: "throttled_input_video"
|
||||
}
|
||||
|
||||
# Crops the image from the center to the size WIDTHxHEIGHT.
|
||||
node: {
|
||||
calculator: "ImageCroppingCalculator"
|
||||
input_stream: "IMAGE_GPU:throttled_input_video"
|
||||
output_stream: "IMAGE_GPU:throttled_input_video_4x3"
|
||||
input_stream: "WIDTH:input_width"
|
||||
input_stream: "HEIGHT:input_height"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.ImageCroppingCalculatorOptions] {
|
||||
border_mode: BORDER_REPLICATE
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Caches a box-presence decision fed back from boxLandmarkSubgraph, and upon
|
||||
# the arrival of the next input image sends out the cached decision with the
|
||||
# timestamp replaced by that of the input image, essentially generating a packet
|
||||
# that carries the previous box-presence decision. Note that upon the arrival
|
||||
# of the very first input image, an empty packet is sent out to jump start the
|
||||
# feedback loop.
|
||||
node {
|
||||
calculator: "PreviousLoopbackCalculator"
|
||||
input_stream: "MAIN:throttled_input_video_4x3"
|
||||
input_stream: "LOOP:box_presence"
|
||||
input_stream_info: {
|
||||
tag_index: "LOOP"
|
||||
back_edge: true
|
||||
}
|
||||
output_stream: "PREV_LOOP:prev_box_presence"
|
||||
}
|
||||
|
||||
# Drops the incoming image if boxLandmarkSubgraph was able to identify box
|
||||
# presence in the previous image. Otherwise, passes the incoming image through
|
||||
# to trigger a new round of box detection in boxDetectionSubgraph.
|
||||
node {
|
||||
calculator: "GateCalculator"
|
||||
input_stream: "throttled_input_video_4x3"
|
||||
input_stream: "DISALLOW:prev_box_presence"
|
||||
output_stream: "detection_input_video"
|
||||
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.GateCalculatorOptions] {
|
||||
empty_packets_as_allow: true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Subgraph that performs 2D object detection.
|
||||
node {
|
||||
calculator: "ObjectDetectionOidV4Subgraph"
|
||||
input_stream: "detection_input_video"
|
||||
input_side_packet: "allowed_labels"
|
||||
output_stream: "NORM_RECT:box_rect_from_object_detections"
|
||||
}
|
||||
|
||||
# Subgraph that localizes box landmarks (see subgraphs/box_landmark_gpu.pbtxt).
|
||||
node {
|
||||
calculator: "BoxLandmarkSubgraph"
|
||||
input_stream: "IMAGE:throttled_input_video_4x3"
|
||||
input_stream: "NORM_RECT:box_rect"
|
||||
output_stream: "FRAME_ANNOTATION:lifted_objects"
|
||||
output_stream: "NORM_RECT:box_rect_from_landmarks"
|
||||
output_stream: "PRESENCE:box_presence"
|
||||
}
|
||||
|
||||
# Caches a box rectangle fed back from boxLandmarkSubgraph, and upon the
|
||||
# arrival of the next input image sends out the cached rectangle with the
|
||||
# timestamp replaced by that of the input image, essentially generating a packet
|
||||
# that carries the previous box rectangle. Note that upon the arrival of the
|
||||
# very first input image, an empty packet is sent out to jump start the
|
||||
# feedback loop.
|
||||
node {
|
||||
calculator: "PreviousLoopbackCalculator"
|
||||
input_stream: "MAIN:throttled_input_video_4x3"
|
||||
input_stream: "LOOP:box_rect_from_landmarks"
|
||||
input_stream_info: {
|
||||
tag_index: "LOOP"
|
||||
back_edge: true
|
||||
}
|
||||
output_stream: "PREV_LOOP:prev_box_rect_from_landmarks"
|
||||
}
|
||||
|
||||
# Merges a stream of box rectangles generated by boxDetectionSubgraph and that
|
||||
# generated by boxLandmarkSubgraph into a single output stream by selecting
|
||||
# between one of the two streams. The former is selected if the incoming packet
|
||||
# is not empty, i.e., box detection is performed on the current image by
|
||||
# boxDetectionSubgraph (because boxLandmarkSubgraph could not identify box
|
||||
# presence in the previous image). Otherwise, the latter is selected, which is
|
||||
# never empty because boxLandmarkSubgraphs processes all images (that went
|
||||
# through FlowLimiterCaculator).
|
||||
node {
|
||||
calculator: "MergeCalculator"
|
||||
input_stream: "box_rect_from_object_detections"
|
||||
input_stream: "prev_box_rect_from_landmarks"
|
||||
output_stream: "box_rect"
|
||||
}
|
||||
|
||||
# The rendering nodes:
|
||||
# We are rendering two meshes: 1) a 3D bounding box, which we overlay directly
|
||||
# on the texture, and 2) a virtual object, which we use as an occlusion mask.
|
||||
# These models are designed using different tools, so we supply a transformation
|
||||
# to bring both of them to the Objectron's coordinate system.
|
||||
|
||||
# Creates a model matrices for the tracked object given the lifted 3D points.
|
||||
# This calculator does two things: 1) Estimates object's pose (orientation,
|
||||
# translation, and scale) from the 3D vertices, and
|
||||
# 2) bring the object from the objectron's coordinate system to the renderer
|
||||
# (OpenGL) coordinate system. Since the final goal is to render a mesh file on
|
||||
# top of the object, we also supply a transformation to bring the mesh to the
|
||||
# objectron's coordinate system, and rescale mesh to the unit size.
|
||||
node {
|
||||
calculator: "AnnotationsToModelMatricesCalculator"
|
||||
input_stream: "ANNOTATIONS:lifted_objects"
|
||||
output_stream: "MODEL_MATRICES:model_matrices"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.AnnotationsToModelMatricesCalculatorOptions] {
|
||||
# Re-scale the CAD model to the size of a unit box
|
||||
model_scale: [0.04, 0.04, 0.04]
|
||||
# Bring the box CAD model to objectron's coordinate system. This
|
||||
# is equivalent of -pi/2 rotation along the y-axis (right-hand rule):
|
||||
# Eigen::AngleAxisf(-M_PI / 2., Eigen::Vector3f::UnitY())
|
||||
model_transformation: [0.0, 0.0, -1.0, 0.0]
|
||||
model_transformation: [0.0, 1.0, 0.0, 0.0]
|
||||
model_transformation: [1.0, 0.0, 0.0, 0.0]
|
||||
model_transformation: [0.0, 0.0, 0.0, 1.0]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Compute the model matrices for the CAD model of the virtual object, to be used
|
||||
# as an occlusion mask. The model will be rendered at the exact same location as
|
||||
# the bounding box.
|
||||
node {
|
||||
calculator: "AnnotationsToModelMatricesCalculator"
|
||||
input_stream: "ANNOTATIONS:lifted_objects"
|
||||
input_side_packet: "MODEL_SCALE:model_scale"
|
||||
input_side_packet: "MODEL_TRANSFORMATION:model_transformation"
|
||||
output_stream: "MODEL_MATRICES:mask_model_matrices"
|
||||
}
|
||||
|
||||
# Render everything together. First we render the 3D bounding box animation,
|
||||
# then we render the occlusion mask.
|
||||
node: {
|
||||
calculator: "GlAnimationOverlayCalculator"
|
||||
input_stream: "VIDEO:throttled_input_video_4x3"
|
||||
input_stream: "MODEL_MATRICES:model_matrices"
|
||||
input_stream: "MASK_MODEL_MATRICES:mask_model_matrices"
|
||||
output_stream: "output_video"
|
||||
input_side_packet: "TEXTURE:box_texture"
|
||||
input_side_packet: "ANIMATION_ASSET:box_asset_name"
|
||||
input_side_packet: "MASK_TEXTURE:obj_texture"
|
||||
input_side_packet: "MASK_ASSET:obj_asset_name"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.GlAnimationOverlayCalculatorOptions] {
|
||||
aspect_ratio: 0.75
|
||||
vertical_fov_degrees: 70.
|
||||
animation_speed_fps: 25
|
||||
}
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -99,7 +99,7 @@ node {
|
||||
[type.googleapis.com/mediapipe.AnnotationsToModelMatricesCalculatorOptions] {
|
||||
# Re-scale the CAD model to the size of a unit box
|
||||
model_scale: [0.15, 0.1, 0.15]
|
||||
# Bring the shoe CAD model to Deep Pursuit 3D's coordinate system. This
|
||||
# Bring the CAD model to Deep Pursuit 3D's coordinate system. This
|
||||
# is equivalent of -pi/2 rotation along the x-axis:
|
||||
# Eigen::AngleAxisf(-M_PI / 2., Eigen::Vector3f::UnitX())
|
||||
model_transformation: [1.0, 0.0, 0.0, 0.0]
|
||||
@@ -1,134 +0,0 @@
|
||||
# MediaPipe object detection 3D with tracking graph.
|
||||
|
||||
# Images on GPU coming into and out of the graph.
|
||||
input_stream: "input_video"
|
||||
input_stream: "input_width"
|
||||
input_stream: "input_height"
|
||||
output_stream: "output_video"
|
||||
|
||||
# Crops the image from the center to the size WIDTHxHEIGHT.
|
||||
node: {
|
||||
calculator: "ImageCroppingCalculator"
|
||||
input_stream: "IMAGE_GPU:input_video"
|
||||
output_stream: "IMAGE_GPU:input_video_4x3"
|
||||
input_stream: "WIDTH:input_width"
|
||||
input_stream: "HEIGHT:input_height"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.ImageCroppingCalculatorOptions] {
|
||||
border_mode: BORDER_REPLICATE
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Creates a copy of the input_video stream. At the end of the graph, the
|
||||
# GlAnimationOverlayCalculator will consume the input_video texture and draws
|
||||
# on top of it.
|
||||
node: {
|
||||
calculator: "GlScalerCalculator"
|
||||
input_stream: "VIDEO:input_video_4x3"
|
||||
output_stream: "VIDEO:input_video_copy"
|
||||
}
|
||||
|
||||
# Resamples the images by specific frame rate. This calculator is used to
|
||||
# control the frequecy of subsequent calculators/subgraphs, e.g. less power
|
||||
# consumption for expensive process.
|
||||
node {
|
||||
calculator: "PacketResamplerCalculator"
|
||||
input_stream: "DATA:input_video_copy"
|
||||
output_stream: "DATA:sampled_input_video"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.PacketResamplerCalculatorOptions] {
|
||||
frame_rate: 5
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
node {
|
||||
calculator: "ObjectronDetectionSubgraphGpu"
|
||||
input_stream: "IMAGE_GPU:sampled_input_video"
|
||||
output_stream: "ANNOTATIONS:objects"
|
||||
}
|
||||
|
||||
node {
|
||||
calculator: "ObjectronTrackingSubgraphGpu"
|
||||
input_stream: "FRAME_ANNOTATION:objects"
|
||||
input_stream: "IMAGE_GPU:input_video_copy"
|
||||
output_stream: "LIFTED_FRAME_ANNOTATION:lifted_tracked_objects"
|
||||
}
|
||||
|
||||
# The rendering nodes:
|
||||
# We are rendering two meshes: 1) a 3D bounding box, which we overlay directly
|
||||
# on the texture, and 2) a shoe CAD model, which we use as an occlusion mask.
|
||||
# These models are designed using different tools, so we supply a transformation
|
||||
# to bring both of them to the Objectron's coordinate system.
|
||||
|
||||
# Creates a model matrices for the tracked object given the lifted 3D points.
|
||||
# This calculator does two things: 1) Estimates object's pose (orientation,
|
||||
# translation, and scale) from the 3D vertices, and
|
||||
# 2) bring the object from the objectron's coordinate system to the renderer
|
||||
# (OpenGL) coordinate system. Since the final goal is to render a mesh file on
|
||||
# top of the object, we also supply a transformation to bring the mesh to the
|
||||
# objectron's coordinate system, and rescale mesh to the unit size.
|
||||
node {
|
||||
calculator: "AnnotationsToModelMatricesCalculator"
|
||||
input_stream: "ANNOTATIONS:lifted_tracked_objects"
|
||||
output_stream: "MODEL_MATRICES:model_matrices"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.AnnotationsToModelMatricesCalculatorOptions] {
|
||||
# Re-scale the CAD model to the size of a unit box
|
||||
model_scale: [0.05, 0.05, 0.05]
|
||||
# Bring the box CAD model to objectron's coordinate system. This
|
||||
# is equivalent of -pi/2 rotation along the y-axis (right-hand rule):
|
||||
# Eigen::AngleAxisf(-M_PI / 2., Eigen::Vector3f::UnitY())
|
||||
model_transformation: [0.0, 0.0, -1.0, 0.0]
|
||||
model_transformation: [0.0, 1.0, 0.0, 0.0]
|
||||
model_transformation: [1.0, 0.0, 0.0, 0.0]
|
||||
model_transformation: [0.0, 0.0, 0.0, 1.0]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Compute the model matrices for the CAD model of the shoe, to be used as an
|
||||
# occlusion mask. The model will be rendered at the exact same location as the
|
||||
# bounding box.
|
||||
node {
|
||||
calculator: "AnnotationsToModelMatricesCalculator"
|
||||
input_stream: "ANNOTATIONS:lifted_tracked_objects"
|
||||
output_stream: "MODEL_MATRICES:mask_model_matrices"
|
||||
#input_side_packet: "MODEL_SCALE:model_scale"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.AnnotationsToModelMatricesCalculatorOptions] {
|
||||
# Re-scale the CAD model to the size of a unit box
|
||||
model_scale: [0.45, 0.25, 0.15]
|
||||
# Bring the shoe CAD model to Deep Pursuit 3D's coordinate system. This
|
||||
# is equivalent of -pi/2 rotation along the x-axis (right-hand rule):
|
||||
# Eigen::AngleAxisf(-M_PI / 2., Eigen::Vector3f::UnitX())
|
||||
model_transformation: [1.0, 0.0, 0.0, 0.0]
|
||||
model_transformation: [0.0, 0.0, 1.0, 0.0]
|
||||
model_transformation: [0.0, -1.0, 0.0, 0.0]
|
||||
model_transformation: [0.0, 0.0, 0.0, 1.0]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Render everything together. First we render the 3D bounding box animation,
|
||||
# then we render the occlusion mask.
|
||||
node: {
|
||||
calculator: "GlAnimationOverlayCalculator"
|
||||
input_stream: "VIDEO:input_video_4x3"
|
||||
input_stream: "MODEL_MATRICES:model_matrices"
|
||||
input_stream: "MASK_MODEL_MATRICES:mask_model_matrices"
|
||||
output_stream: "output_video"
|
||||
input_side_packet: "TEXTURE:box_texture"
|
||||
input_side_packet: "ANIMATION_ASSET:box_asset_name"
|
||||
input_side_packet: "MASK_TEXTURE:obj_texture"
|
||||
input_side_packet: "MASK_ASSET:obj_asset_name"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.GlAnimationOverlayCalculatorOptions] {
|
||||
# Output resolution is 480x640 with the aspect ratio of 0.75
|
||||
aspect_ratio: 0.75
|
||||
vertical_fov_degrees: 70.
|
||||
animation_speed_fps: 25
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -50,3 +50,48 @@ mediapipe_simple_subgraph(
|
||||
"//mediapipe/graphs/object_detection_3d/calculators:lift_2d_frame_annotation_to_3d_calculator",
|
||||
],
|
||||
)
|
||||
|
||||
mediapipe_simple_subgraph(
|
||||
name = "box_landmark_gpu",
|
||||
graph = "box_landmark_gpu.pbtxt",
|
||||
register_as = "BoxLandmarkSubgraph",
|
||||
deps = [
|
||||
"//mediapipe/calculators/core:split_vector_calculator",
|
||||
"//mediapipe/calculators/image:image_cropping_calculator",
|
||||
"//mediapipe/calculators/image:image_properties_calculator",
|
||||
"//mediapipe/calculators/image:image_transformation_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_converter_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_custom_op_resolver_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_inference_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_tensors_to_floats_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_tensors_to_landmarks_calculator",
|
||||
"//mediapipe/calculators/util:detections_to_rects_calculator",
|
||||
"//mediapipe/calculators/util:landmark_letterbox_removal_calculator",
|
||||
"//mediapipe/calculators/util:landmark_projection_calculator",
|
||||
"//mediapipe/calculators/util:landmarks_smoothing_calculator",
|
||||
"//mediapipe/calculators/util:landmarks_to_detection_calculator",
|
||||
"//mediapipe/calculators/util:rect_transformation_calculator",
|
||||
"//mediapipe/calculators/util:thresholding_calculator",
|
||||
"//mediapipe/graphs/object_detection_3d/calculators:frame_annotation_to_rect_calculator",
|
||||
"//mediapipe/graphs/object_detection_3d/calculators:landmarks_to_frame_annotation_calculator",
|
||||
"//mediapipe/graphs/object_detection_3d/calculators:lift_2d_frame_annotation_to_3d_calculator",
|
||||
],
|
||||
)
|
||||
|
||||
mediapipe_simple_subgraph(
|
||||
name = "object_detection_oid_v4_gpu",
|
||||
graph = "object_detection_oid_v4_gpu.pbtxt",
|
||||
register_as = "ObjectDetectionOidV4Subgraph",
|
||||
deps = [
|
||||
"//mediapipe/calculators/image:image_transformation_calculator",
|
||||
"//mediapipe/calculators/tflite:ssd_anchors_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_converter_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_inference_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_tensors_to_detections_calculator",
|
||||
"//mediapipe/calculators/util:detection_label_id_to_text_calculator",
|
||||
"//mediapipe/calculators/util:detections_to_rects_calculator",
|
||||
"//mediapipe/calculators/util:non_max_suppression_calculator",
|
||||
"//mediapipe/calculators/util:rect_transformation_calculator",
|
||||
"//mediapipe/graphs/object_detection_3d/calculators:filter_detection_calculator",
|
||||
],
|
||||
)
|
||||
|
||||
@@ -0,0 +1,205 @@
|
||||
# MediaPipe Box landmark localization subgraph.
|
||||
|
||||
type: "BoxLandmarkSubgraph"
|
||||
|
||||
input_stream: "IMAGE:input_video"
|
||||
input_stream: "NORM_RECT:box_rect"
|
||||
output_stream: "FRAME_ANNOTATION:lifted_box"
|
||||
output_stream: "NORM_RECT:box_rect_for_next_frame"
|
||||
output_stream: "PRESENCE:box_presence"
|
||||
|
||||
# Crops the rectangle that contains a box from the input image.
|
||||
node {
|
||||
calculator: "ImageCroppingCalculator"
|
||||
input_stream: "IMAGE_GPU:input_video"
|
||||
input_stream: "NORM_RECT:box_rect"
|
||||
output_stream: "IMAGE_GPU:box_image"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.ImageCroppingCalculatorOptions] {
|
||||
border_mode: BORDER_REPLICATE
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Transforms the input image on GPU to a 256x256 image. To scale the input
|
||||
# image, the scale_mode option is set to FIT to preserve the aspect ratio,
|
||||
# resulting in potential letterboxing in the transformed image.
|
||||
node: {
|
||||
calculator: "ImageTransformationCalculator"
|
||||
input_stream: "IMAGE_GPU:box_image"
|
||||
output_stream: "IMAGE_GPU:transformed_box_image"
|
||||
output_stream: "LETTERBOX_PADDING:letterbox_padding"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.ImageTransformationCalculatorOptions] {
|
||||
output_width: 224
|
||||
output_height: 224
|
||||
scale_mode: FIT
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Converts the transformed input image on GPU into an image tensor stored as a
|
||||
# TfLiteTensor.
|
||||
node {
|
||||
calculator: "TfLiteConverterCalculator"
|
||||
input_stream: "IMAGE_GPU:transformed_box_image"
|
||||
output_stream: "TENSORS_GPU:image_tensor"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.TfLiteConverterCalculatorOptions] {
|
||||
zero_center: false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Generates a single side packet containing a TensorFlow Lite op resolver that
|
||||
# supports custom ops needed by the model used in this graph.
|
||||
node {
|
||||
calculator: "TfLiteCustomOpResolverCalculator"
|
||||
output_side_packet: "opresolver"
|
||||
}
|
||||
|
||||
# Runs a TensorFlow Lite model on GPU that takes an image tensor and outputs a
|
||||
# vector of tensors representing, for instance, detection boxes/keypoints and
|
||||
# scores.
|
||||
node {
|
||||
calculator: "TfLiteInferenceCalculator"
|
||||
input_stream: "TENSORS_GPU:image_tensor"
|
||||
output_stream: "TENSORS:output_tensors"
|
||||
input_side_packet: "CUSTOM_OP_RESOLVER:opresolver"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.TfLiteInferenceCalculatorOptions] {
|
||||
model_path: "object_detection_3d.tflite"
|
||||
use_gpu: true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Splits a vector of tensors into multiple vectors.
|
||||
node {
|
||||
calculator: "SplitTfLiteTensorVectorCalculator"
|
||||
input_stream: "output_tensors"
|
||||
output_stream: "landmark_tensors"
|
||||
output_stream: "box_flag_tensor"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.SplitVectorCalculatorOptions] {
|
||||
ranges: { begin: 0 end: 1 }
|
||||
ranges: { begin: 1 end: 2 }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Converts the box-flag tensor into a float that represents the confidence
|
||||
# score of box presence.
|
||||
node {
|
||||
calculator: "TfLiteTensorsToFloatsCalculator"
|
||||
input_stream: "TENSORS:box_flag_tensor"
|
||||
output_stream: "FLOAT:box_presence_score"
|
||||
}
|
||||
|
||||
# Applies a threshold to the confidence score to determine whether a box is
|
||||
# present.
|
||||
node {
|
||||
calculator: "ThresholdingCalculator"
|
||||
input_stream: "FLOAT:box_presence_score"
|
||||
output_stream: "FLAG:box_presence"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.ThresholdingCalculatorOptions] {
|
||||
threshold: 0.99
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Decodes the landmark tensors into a list of landmarks, where the landmark
|
||||
# coordinates are normalized by the size of the input image to the model.
|
||||
node {
|
||||
calculator: "TfLiteTensorsToLandmarksCalculator"
|
||||
input_stream: "TENSORS:landmark_tensors"
|
||||
output_stream: "NORM_LANDMARKS:landmarks"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.TfLiteTensorsToLandmarksCalculatorOptions] {
|
||||
num_landmarks: 9
|
||||
input_image_width: 224
|
||||
input_image_height: 224
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Adjusts landmarks (already normalized to [0.f, 1.f]) on the letterboxed box
|
||||
# image (after image transformation with the FIT scale mode) to the
|
||||
# corresponding locations on the same image with the letterbox removed (box
|
||||
# image before image transformation).
|
||||
node {
|
||||
calculator: "LandmarkLetterboxRemovalCalculator"
|
||||
input_stream: "LANDMARKS:landmarks"
|
||||
input_stream: "LETTERBOX_PADDING:letterbox_padding"
|
||||
output_stream: "LANDMARKS:scaled_landmarks"
|
||||
}
|
||||
|
||||
# Projects the landmarks from the cropped box image to the corresponding
|
||||
# locations on the full image before cropping (input to the graph).
|
||||
node {
|
||||
calculator: "LandmarkProjectionCalculator"
|
||||
input_stream: "NORM_LANDMARKS:scaled_landmarks"
|
||||
input_stream: "NORM_RECT:box_rect"
|
||||
output_stream: "NORM_LANDMARKS:box_landmarks"
|
||||
}
|
||||
|
||||
# Extracts image size from the input images.
|
||||
node {
|
||||
calculator: "ImagePropertiesCalculator"
|
||||
input_stream: "IMAGE_GPU:input_video"
|
||||
output_stream: "SIZE:image_size"
|
||||
}
|
||||
|
||||
# Smooth predicted landmarks coordinates.
|
||||
node {
|
||||
calculator: "LandmarksSmoothingCalculator"
|
||||
input_stream: "NORM_LANDMARKS:box_landmarks"
|
||||
input_stream: "IMAGE_SIZE:image_size"
|
||||
output_stream: "NORM_FILTERED_LANDMARKS:box_landmarks_filtered"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.LandmarksSmoothingCalculatorOptions] {
|
||||
velocity_filter: {
|
||||
window_size: 10
|
||||
velocity_scale: 7.5
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Convert box landmarks to frame annotation.
|
||||
node {
|
||||
calculator: "LandmarksToFrameAnnotationCalculator"
|
||||
input_stream: "LANDMARKS:box_landmarks_filtered"
|
||||
output_stream: "FRAME_ANNOTATION:box_annotation"
|
||||
}
|
||||
|
||||
# Lift the 2D landmarks to 3D using EPnP algorithm.
|
||||
node {
|
||||
calculator: "Lift2DFrameAnnotationTo3DCalculator"
|
||||
input_stream: "FRAME_ANNOTATION:box_annotation"
|
||||
output_stream: "LIFTED_FRAME_ANNOTATION:lifted_box"
|
||||
}
|
||||
|
||||
# Get rotated rectangle from lifted box.
|
||||
node {
|
||||
calculator: "FrameAnnotationToRectCalculator"
|
||||
input_stream: "FRAME_ANNOTATION:lifted_box"
|
||||
output_stream: "NORM_RECT:rect_from_box"
|
||||
}
|
||||
|
||||
# Expands the box rectangle so that in the next video frame it's likely to
|
||||
# still contain the box even with some motion.
|
||||
node {
|
||||
calculator: "RectTransformationCalculator"
|
||||
input_stream: "NORM_RECT:rect_from_box"
|
||||
input_stream: "IMAGE_SIZE:image_size"
|
||||
output_stream: "box_rect_for_next_frame"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.RectTransformationCalculatorOptions] {
|
||||
scale_x: 1.5
|
||||
scale_y: 1.5
|
||||
square_long: true
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,177 @@
|
||||
# MediaPipe Objectron object bounding box detection subgraph.
|
||||
|
||||
type: "ObjectDetectionSubgraph"
|
||||
|
||||
input_stream: "input_video"
|
||||
input_side_packet: "allowed_labels"
|
||||
output_stream: "NORM_RECT:box_rect_from_object_detections"
|
||||
|
||||
# Transforms the input image on GPU to a 320x320 image. To scale the image, by
|
||||
# default it uses the STRETCH scale mode that maps the entire input image to the
|
||||
# entire transformed image. As a result, image aspect ratio may be changed and
|
||||
# objects in the image may be deformed (stretched or squeezed), but the object
|
||||
# detection model used in this graph is agnostic to that deformation.
|
||||
node: {
|
||||
calculator: "ImageTransformationCalculator"
|
||||
input_stream: "IMAGE_GPU:input_video"
|
||||
output_stream: "IMAGE_GPU:transformed_input_video"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.ImageTransformationCalculatorOptions] {
|
||||
output_width: 300
|
||||
output_height: 300
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Converts the transformed input image on GPU into an image tensor stored as a
|
||||
# TfLiteTensor.
|
||||
node {
|
||||
calculator: "TfLiteConverterCalculator"
|
||||
input_stream: "IMAGE_GPU:transformed_input_video"
|
||||
output_stream: "TENSORS_GPU:image_tensor"
|
||||
}
|
||||
|
||||
# Runs a TensorFlow Lite model on GPU that takes an image tensor and outputs a
|
||||
# vector of tensors representing, for instance, detection boxes/keypoints and
|
||||
# scores.
|
||||
node {
|
||||
calculator: "TfLiteInferenceCalculator"
|
||||
input_stream: "TENSORS_GPU:image_tensor"
|
||||
output_stream: "TENSORS_GPU:detection_tensors"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.TfLiteInferenceCalculatorOptions] {
|
||||
model_path: "object_detection_ssd_mobilenetv2_oidv4_fp16.tflite"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Generates a single side packet containing a vector of SSD anchors based on
|
||||
# the specification in the options.
|
||||
node {
|
||||
calculator: "SsdAnchorsCalculator"
|
||||
output_side_packet: "anchors"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.SsdAnchorsCalculatorOptions] {
|
||||
num_layers: 6
|
||||
min_scale: 0.2
|
||||
max_scale: 0.95
|
||||
input_size_height: 300
|
||||
input_size_width: 300
|
||||
anchor_offset_x: 0.5
|
||||
anchor_offset_y: 0.5
|
||||
strides: 16
|
||||
strides: 32
|
||||
strides: 64
|
||||
strides: 128
|
||||
strides: 256
|
||||
strides: 512
|
||||
aspect_ratios: 1.0
|
||||
aspect_ratios: 2.0
|
||||
aspect_ratios: 0.5
|
||||
aspect_ratios: 3.0
|
||||
aspect_ratios: 0.3333
|
||||
reduce_boxes_in_lowest_layer: true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Decodes the detection tensors generated by the TensorFlow Lite model, based on
|
||||
# the SSD anchors and the specification in the options, into a vector of
|
||||
# detections. Each detection describes a detected object.
|
||||
node {
|
||||
calculator: "TfLiteTensorsToDetectionsCalculator"
|
||||
input_stream: "TENSORS_GPU:detection_tensors"
|
||||
input_side_packet: "ANCHORS:anchors"
|
||||
output_stream: "DETECTIONS:detections"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.TfLiteTensorsToDetectionsCalculatorOptions] {
|
||||
num_classes: 195
|
||||
num_boxes: 1917
|
||||
num_coords: 4
|
||||
ignore_classes: 0
|
||||
sigmoid_score: true
|
||||
apply_exponential_on_box_size: true
|
||||
x_scale: 10.0
|
||||
y_scale: 10.0
|
||||
h_scale: 5.0
|
||||
w_scale: 5.0
|
||||
min_score_thresh: 0.6
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Performs non-max suppression to remove excessive detections.
|
||||
node {
|
||||
calculator: "NonMaxSuppressionCalculator"
|
||||
input_stream: "detections"
|
||||
output_stream: "suppressed_detections"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.NonMaxSuppressionCalculatorOptions] {
|
||||
min_suppression_threshold: 0.4
|
||||
max_num_detections: 1
|
||||
overlap_type: INTERSECTION_OVER_UNION
|
||||
return_empty_detections: true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Maps detection label IDs to the corresponding label text. The label map is
|
||||
# provided in the label_map_path option.
|
||||
node {
|
||||
calculator: "DetectionLabelIdToTextCalculator"
|
||||
input_stream: "suppressed_detections"
|
||||
output_stream: "labeled_detections"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.DetectionLabelIdToTextCalculatorOptions] {
|
||||
label_map_path: "object_detection_oidv4_labelmap.pbtxt"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
node {
|
||||
calculator: "FilterDetectionCalculator"
|
||||
input_stream: "DETECTIONS:labeled_detections"
|
||||
output_stream: "DETECTIONS:filtered_detections"
|
||||
input_side_packet: "LABELS_CSV:allowed_labels"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.FilterDetectionCalculatorOptions]: {
|
||||
min_score: 0.4
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Extracts image size from the input images.
|
||||
node {
|
||||
calculator: "ImagePropertiesCalculator"
|
||||
input_stream: "IMAGE_GPU:input_video"
|
||||
output_stream: "SIZE:image_size"
|
||||
}
|
||||
|
||||
# Converts results of box detection into a rectangle (normalized by image size)
|
||||
# that encloses the box.
|
||||
node {
|
||||
calculator: "DetectionsToRectsCalculator"
|
||||
input_stream: "DETECTIONS:filtered_detections"
|
||||
input_stream: "IMAGE_SIZE:image_size"
|
||||
output_stream: "NORM_RECT:box_rect"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.DetectionsToRectsCalculatorOptions] {
|
||||
output_zero_rect_for_empty_detections: true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Expands the rectangle that contains the box so that it's likely to cover the
|
||||
# entire box.
|
||||
node {
|
||||
calculator: "RectTransformationCalculator"
|
||||
input_stream: "NORM_RECT:box_rect"
|
||||
input_stream: "IMAGE_SIZE:image_size"
|
||||
output_stream: "box_rect_from_object_detections"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.RectTransformationCalculatorOptions] {
|
||||
scale_x: 1.5
|
||||
scale_y: 1.5
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user