Project import generated by Copybara.
GitOrigin-RevId: 5aca6b3f07b67e09988a901f50f595ca5f566e67
This commit is contained in:
MediaPipe Team
@@ -1,6 +1,6 @@
|
||||
# MediaPipe graph that performs face detection with TensorFlow Lite on CPU.
|
||||
# Used in the examples in
|
||||
# mediapipie/examples/desktop/face_detection:face_detection_cpu.
|
||||
# mediapipe/examples/desktop/face_detection:face_detection_cpu.
|
||||
|
||||
# Images on GPU coming into and out of the graph.
|
||||
input_stream: "input_video"
|
||||
|
||||
@@ -12,26 +12,33 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
licenses(["notice"]) # Apache 2.0
|
||||
|
||||
package(default_visibility = ["//visibility:public"])
|
||||
|
||||
load(
|
||||
"//mediapipe/framework/tool:mediapipe_graph.bzl",
|
||||
"mediapipe_binary_graph",
|
||||
)
|
||||
|
||||
licenses(["notice"]) # Apache 2.0
|
||||
|
||||
package(default_visibility = ["//visibility:public"])
|
||||
|
||||
cc_library(
|
||||
name = "desktop_tflite_calculators",
|
||||
name = "desktop_offline_calculators",
|
||||
deps = [
|
||||
"//mediapipe/calculators/core:flow_limiter_calculator",
|
||||
"//mediapipe/calculators/core:gate_calculator",
|
||||
"//mediapipe/calculators/core:immediate_mux_calculator",
|
||||
"//mediapipe/calculators/core:merge_calculator",
|
||||
"//mediapipe/calculators/core:packet_inner_join_calculator",
|
||||
"//mediapipe/calculators/core:previous_loopback_calculator",
|
||||
"//mediapipe/calculators/video:opencv_video_decoder_calculator",
|
||||
"//mediapipe/calculators/video:opencv_video_encoder_calculator",
|
||||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "desktop_tflite_calculators",
|
||||
deps = [
|
||||
":desktop_offline_calculators",
|
||||
"//mediapipe/calculators/core:merge_calculator",
|
||||
"//mediapipe/graphs/hand_tracking/subgraphs:hand_detection_cpu",
|
||||
"//mediapipe/graphs/hand_tracking/subgraphs:hand_landmark_cpu",
|
||||
"//mediapipe/graphs/hand_tracking/subgraphs:renderer_cpu",
|
||||
@@ -58,6 +65,39 @@ mediapipe_binary_graph(
|
||||
deps = [":mobile_calculators"],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "multi_hand_desktop_tflite_calculators",
|
||||
deps = [
|
||||
":desktop_offline_calculators",
|
||||
"//mediapipe/calculators/util:association_norm_rect_calculator",
|
||||
"//mediapipe/calculators/util:collection_has_min_size_calculator",
|
||||
"//mediapipe/graphs/hand_tracking/subgraphs:multi_hand_detection_cpu",
|
||||
"//mediapipe/graphs/hand_tracking/subgraphs:multi_hand_landmark_cpu",
|
||||
"//mediapipe/graphs/hand_tracking/subgraphs:multi_hand_renderer_cpu",
|
||||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "multi_hand_mobile_calculators",
|
||||
deps = [
|
||||
"//mediapipe/calculators/core:flow_limiter_calculator",
|
||||
"//mediapipe/calculators/core:gate_calculator",
|
||||
"//mediapipe/calculators/core:previous_loopback_calculator",
|
||||
"//mediapipe/calculators/util:association_norm_rect_calculator",
|
||||
"//mediapipe/calculators/util:collection_has_min_size_calculator",
|
||||
"//mediapipe/graphs/hand_tracking/subgraphs:multi_hand_detection_gpu",
|
||||
"//mediapipe/graphs/hand_tracking/subgraphs:multi_hand_landmark_gpu",
|
||||
"//mediapipe/graphs/hand_tracking/subgraphs:multi_hand_renderer_gpu",
|
||||
],
|
||||
)
|
||||
|
||||
mediapipe_binary_graph(
|
||||
name = "multi_hand_tracking_mobile_gpu_binary_graph",
|
||||
graph = "multi_hand_tracking_mobile.pbtxt",
|
||||
output_name = "multi_hand_tracking_mobile_gpu.binarypb",
|
||||
deps = [":multi_hand_mobile_calculators"],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "detection_mobile_calculators",
|
||||
deps = [
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# MediaPipe graph that performs hand detection on desktop with TensorFlow Lite
|
||||
# on CPU.
|
||||
# Used in the example in
|
||||
# mediapipie/examples/desktop/hand_tracking:hand_detection_cpu.
|
||||
# mediapipe/examples/desktop/hand_tracking:hand_detection_cpu.
|
||||
|
||||
# Images coming into and out of the graph.
|
||||
input_stream: "input_video"
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# MediaPipe graph that performs hand detection with TensorFlow Lite on GPU.
|
||||
# Used in the examples in
|
||||
# mediapipie/examples/android/src/java/com/mediapipe/apps/handdetectiongpu and
|
||||
# mediapipie/examples/ios/handdetectiongpu.
|
||||
# mediapipe/examples/android/src/java/com/mediapipe/apps/handdetectiongpu and
|
||||
# mediapipe/examples/ios/handdetectiongpu.
|
||||
|
||||
# Images coming into and out of the graph.
|
||||
input_stream: "input_video"
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# MediaPipe graph that performs hand tracking on desktop with TensorFlow Lite
|
||||
# on CPU.
|
||||
# Used in the example in
|
||||
# mediapipie/examples/desktop/hand_tracking:hand_tracking_tflite.
|
||||
# mediapipe/examples/desktop/hand_tracking:hand_tracking_tflite.
|
||||
|
||||
# max_queue_size limits the number of packets enqueued on any input stream
|
||||
# by throttling inputs to the graph. This makes the graph only process one
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# MediaPipe graph that performs hand tracking with TensorFlow Lite on GPU.
|
||||
# Used in the examples in
|
||||
# mediapipie/examples/android/src/java/com/mediapipe/apps/handtrackinggpu and
|
||||
# mediapipie/examples/ios/handtrackinggpu.
|
||||
# mediapipe/examples/android/src/java/com/mediapipe/apps/handtrackinggpu and
|
||||
# mediapipe/examples/ios/handtrackinggpu.
|
||||
|
||||
# Images coming into and out of the graph.
|
||||
input_stream: "input_video"
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
# MediaPipe graph that performs multi-hand tracking on desktop with TensorFlow
|
||||
# Lite on CPU.
|
||||
# Used in the example in
|
||||
# mediapipie/examples/desktop/hand_tracking:multi_hand_tracking_tflite.
|
||||
|
||||
# max_queue_size limits the number of packets enqueued on any input stream
|
||||
# by throttling inputs to the graph. This makes the graph only process one
|
||||
# frame per time.
|
||||
max_queue_size: 1
|
||||
|
||||
# Decodes an input video file into images and a video header.
|
||||
node {
|
||||
calculator: "OpenCvVideoDecoderCalculator"
|
||||
input_side_packet: "INPUT_FILE_PATH:input_video_path"
|
||||
output_stream: "VIDEO:input_video"
|
||||
output_stream: "VIDEO_PRESTREAM:input_video_header"
|
||||
}
|
||||
|
||||
# Determines if an input vector of NormalizedRect has a size greater than or
|
||||
# equal to the provided min_size.
|
||||
node {
|
||||
calculator: "NormalizedRectVectorHasMinSizeCalculator"
|
||||
input_stream: "ITERABLE:prev_multi_hand_rects_from_landmarks"
|
||||
output_stream: "prev_has_enough_hands"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.CollectionHasMinSizeCalculatorOptions] {
|
||||
# This value can be changed to support tracking arbitrary number of hands.
|
||||
# Please also remember to modify max_vec_size in
|
||||
# ClipVectorSizeCalculatorOptions in
|
||||
# mediapipe/graphs/hand_tracking/subgraphs/multi_hand_detection_cpu.pbtxt
|
||||
min_size: 2
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Drops the incoming image if the previous frame had at least N hands.
|
||||
# Otherwise, passes the incoming image through to trigger a new round of hand
|
||||
# detection in MultiHandDetectionSubgraph.
|
||||
node {
|
||||
calculator: "GateCalculator"
|
||||
input_stream: "input_video"
|
||||
input_stream: "DISALLOW:prev_has_enough_hands"
|
||||
output_stream: "multi_hand_detection_input_video"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.GateCalculatorOptions] {
|
||||
empty_packets_as_allow: true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Subgraph that detections hands (see multi_hand_detection_cpu.pbtxt).
|
||||
node {
|
||||
calculator: "MultiHandDetectionSubgraph"
|
||||
input_stream: "multi_hand_detection_input_video"
|
||||
output_stream: "DETECTIONS:multi_palm_detections"
|
||||
output_stream: "NORM_RECTS:multi_palm_rects"
|
||||
}
|
||||
|
||||
# Subgraph that localizes hand landmarks for multiple hands (see
|
||||
# multi_hand_landmark.pbtxt).
|
||||
node {
|
||||
calculator: "MultiHandLandmarkSubgraph"
|
||||
input_stream: "IMAGE:input_video"
|
||||
input_stream: "NORM_RECTS:multi_hand_rects"
|
||||
output_stream: "LANDMARKS:multi_hand_landmarks"
|
||||
output_stream: "NORM_RECTS:multi_hand_rects_from_landmarks"
|
||||
}
|
||||
|
||||
# Caches a hand rectangle fed back from MultiHandLandmarkSubgraph, and upon the
|
||||
# arrival of the next input image sends out the cached rectangle with the
|
||||
# timestamp replaced by that of the input image, essentially generating a packet
|
||||
# that carries the previous hand rectangle. Note that upon the arrival of the
|
||||
# very first input image, an empty packet is sent out to jump start the
|
||||
# feedback loop.
|
||||
node {
|
||||
calculator: "PreviousLoopbackCalculator"
|
||||
input_stream: "MAIN:input_video"
|
||||
input_stream: "LOOP:multi_hand_rects_from_landmarks"
|
||||
input_stream_info: {
|
||||
tag_index: "LOOP"
|
||||
back_edge: true
|
||||
}
|
||||
output_stream: "PREV_LOOP:prev_multi_hand_rects_from_landmarks"
|
||||
}
|
||||
|
||||
# Performs association between NormalizedRect vector elements from previous
|
||||
# frame and those from the current frame if MultiHandDetectionSubgraph runs.
|
||||
# This calculator ensures that the output multi_hand_rects vector doesn't
|
||||
# contain overlapping regions based on the specified min_similarity_threshold.
|
||||
node {
|
||||
calculator: "AssociationNormRectCalculator"
|
||||
input_stream: "prev_multi_hand_rects_from_landmarks"
|
||||
input_stream: "multi_palm_rects"
|
||||
output_stream: "multi_hand_rects"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.AssociationCalculatorOptions] {
|
||||
min_similarity_threshold: 0.1
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Subgraph that renders annotations and overlays them on top of the input
|
||||
# images (see multi_hand_renderer_cpu.pbtxt).
|
||||
node {
|
||||
calculator: "MultiHandRendererSubgraph"
|
||||
input_stream: "IMAGE:input_video"
|
||||
input_stream: "DETECTIONS:multi_palm_detections"
|
||||
input_stream: "LANDMARKS:multi_hand_landmarks"
|
||||
input_stream: "NORM_RECTS:0:multi_palm_rects"
|
||||
input_stream: "NORM_RECTS:1:multi_hand_rects"
|
||||
output_stream: "IMAGE:output_video"
|
||||
}
|
||||
|
||||
# Encodes the annotated images into a video file, adopting properties specified
|
||||
# in the input video header, e.g., video framerate.
|
||||
node {
|
||||
calculator: "OpenCvVideoEncoderCalculator"
|
||||
input_stream: "VIDEO:output_video"
|
||||
input_stream: "VIDEO_PRESTREAM:input_video_header"
|
||||
input_side_packet: "OUTPUT_FILE_PATH:output_video_path"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.OpenCvVideoEncoderCalculatorOptions]: {
|
||||
codec: "avc1"
|
||||
video_format: "mp4"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
# MediaPipe graph that performs multi-hand tracking on desktop with TensorFlow
|
||||
# Lite on CPU.
|
||||
# Used in the example in
|
||||
# mediapipie/examples/desktop/hand_tracking:multi_hand_tracking_cpu.
|
||||
|
||||
# Images coming into and out of the graph.
|
||||
input_stream: "input_video"
|
||||
output_stream: "output_video"
|
||||
|
||||
# Determines if an input vector of NormalizedRect has a size greater than or
|
||||
# equal to the provided min_size.
|
||||
node {
|
||||
calculator: "NormalizedRectVectorHasMinSizeCalculator"
|
||||
input_stream: "ITERABLE:prev_multi_hand_rects_from_landmarks"
|
||||
output_stream: "prev_has_enough_hands"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.CollectionHasMinSizeCalculatorOptions] {
|
||||
# This value can be changed to support tracking arbitrary number of hands.
|
||||
# Please also remember to modify max_vec_size in
|
||||
# ClipVectorSizeCalculatorOptions in
|
||||
# mediapipe/graphs/hand_tracking/subgraphs/multi_hand_detection_gpu.pbtxt
|
||||
min_size: 2
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Drops the incoming image if the previous frame had at least N hands.
|
||||
# Otherwise, passes the incoming image through to trigger a new round of hand
|
||||
# detection in MultiHandDetectionSubgraph.
|
||||
node {
|
||||
calculator: "GateCalculator"
|
||||
input_stream: "input_video"
|
||||
input_stream: "DISALLOW:prev_has_enough_hands"
|
||||
output_stream: "multi_hand_detection_input_video"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.GateCalculatorOptions] {
|
||||
empty_packets_as_allow: true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Subgraph that detections hands (see multi_hand_detection_cpu.pbtxt).
|
||||
node {
|
||||
calculator: "MultiHandDetectionSubgraph"
|
||||
input_stream: "multi_hand_detection_input_video"
|
||||
output_stream: "DETECTIONS:multi_palm_detections"
|
||||
output_stream: "NORM_RECTS:multi_palm_rects"
|
||||
}
|
||||
|
||||
# Subgraph that localizes hand landmarks for multiple hands (see
|
||||
# multi_hand_landmark.pbtxt).
|
||||
node {
|
||||
calculator: "MultiHandLandmarkSubgraph"
|
||||
input_stream: "IMAGE:input_video"
|
||||
input_stream: "NORM_RECTS:multi_hand_rects"
|
||||
output_stream: "LANDMARKS:multi_hand_landmarks"
|
||||
output_stream: "NORM_RECTS:multi_hand_rects_from_landmarks"
|
||||
}
|
||||
|
||||
# Caches a hand rectangle fed back from MultiHandLandmarkSubgraph, and upon the
|
||||
# arrival of the next input image sends out the cached rectangle with the
|
||||
# timestamp replaced by that of the input image, essentially generating a packet
|
||||
# that carries the previous hand rectangle. Note that upon the arrival of the
|
||||
# very first input image, an empty packet is sent out to jump start the
|
||||
# feedback loop.
|
||||
node {
|
||||
calculator: "PreviousLoopbackCalculator"
|
||||
input_stream: "MAIN:input_video"
|
||||
input_stream: "LOOP:multi_hand_rects_from_landmarks"
|
||||
input_stream_info: {
|
||||
tag_index: "LOOP"
|
||||
back_edge: true
|
||||
}
|
||||
output_stream: "PREV_LOOP:prev_multi_hand_rects_from_landmarks"
|
||||
}
|
||||
|
||||
# Performs association between NormalizedRect vector elements from previous
|
||||
# frame and those from the current frame if MultiHandDetectionSubgraph runs.
|
||||
# This calculator ensures that the output multi_hand_rects vector doesn't
|
||||
# contain overlapping regions based on the specified min_similarity_threshold.
|
||||
node {
|
||||
calculator: "AssociationNormRectCalculator"
|
||||
input_stream: "prev_multi_hand_rects_from_landmarks"
|
||||
input_stream: "multi_palm_rects"
|
||||
output_stream: "multi_hand_rects"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.AssociationCalculatorOptions] {
|
||||
min_similarity_threshold: 0.1
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Subgraph that renders annotations and overlays them on top of the input
|
||||
# images (see multi_hand_renderer_cpu.pbtxt).
|
||||
node {
|
||||
calculator: "MultiHandRendererSubgraph"
|
||||
input_stream: "IMAGE:input_video"
|
||||
input_stream: "DETECTIONS:multi_palm_detections"
|
||||
input_stream: "LANDMARKS:multi_hand_landmarks"
|
||||
input_stream: "NORM_RECTS:0:multi_palm_rects"
|
||||
input_stream: "NORM_RECTS:1:multi_hand_rects"
|
||||
output_stream: "IMAGE:output_video"
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
# MediaPipe graph that performs multi-hand tracking with TensorFlow Lite on GPU.
|
||||
# Used in the examples in
|
||||
# mediapipe/examples/android/src/java/com/mediapipe/apps/multihandtrackinggpu.
|
||||
|
||||
# Images coming into and out of the graph.
|
||||
input_stream: "input_video"
|
||||
output_stream: "output_video"
|
||||
|
||||
# Throttles the images flowing downstream for flow control. It passes through
|
||||
# the very first incoming image unaltered, and waits for downstream nodes
|
||||
# (calculators and subgraphs) in the graph to finish their tasks before it
|
||||
# passes through another image. All images that come in while waiting are
|
||||
# dropped, limiting the number of in-flight images in most part of the graph to
|
||||
# 1. This prevents the downstream nodes from queuing up incoming images and data
|
||||
# excessively, which leads to increased latency and memory usage, unwanted in
|
||||
# real-time mobile applications. It also eliminates unnecessarily computation,
|
||||
# e.g., the output produced by a node may get dropped downstream if the
|
||||
# subsequent nodes are still busy processing previous inputs.
|
||||
node {
|
||||
calculator: "FlowLimiterCalculator"
|
||||
input_stream: "input_video"
|
||||
input_stream: "FINISHED:multi_hand_rects"
|
||||
input_stream_info: {
|
||||
tag_index: "FINISHED"
|
||||
back_edge: true
|
||||
}
|
||||
output_stream: "throttled_input_video"
|
||||
}
|
||||
|
||||
# Determines if an input vector of NormalizedRect has a size greater than or
|
||||
# equal to the provided min_size.
|
||||
node {
|
||||
calculator: "NormalizedRectVectorHasMinSizeCalculator"
|
||||
input_stream: "ITERABLE:prev_multi_hand_rects_from_landmarks"
|
||||
output_stream: "prev_has_enough_hands"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.CollectionHasMinSizeCalculatorOptions] {
|
||||
# This value can be changed to support tracking arbitrary number of hands.
|
||||
# Please also remember to modify max_vec_size in
|
||||
# ClipVectorSizeCalculatorOptions in
|
||||
# mediapipe/graphs/hand_tracking/subgraphs/multi_hand_detection_gpu.pbtxt
|
||||
min_size: 2
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Drops the incoming image if the previous frame had at least N hands.
|
||||
# Otherwise, passes the incoming image through to trigger a new round of hand
|
||||
# detection in MultiHandDetectionSubgraph.
|
||||
node {
|
||||
calculator: "GateCalculator"
|
||||
input_stream: "throttled_input_video"
|
||||
input_stream: "DISALLOW:prev_has_enough_hands"
|
||||
output_stream: "multi_hand_detection_input_video"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.GateCalculatorOptions] {
|
||||
empty_packets_as_allow: true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Subgraph that detections hands (see multi_hand_detection_gpu.pbtxt).
|
||||
node {
|
||||
calculator: "MultiHandDetectionSubgraph"
|
||||
input_stream: "multi_hand_detection_input_video"
|
||||
output_stream: "DETECTIONS:multi_palm_detections"
|
||||
output_stream: "NORM_RECTS:multi_palm_rects"
|
||||
}
|
||||
|
||||
# Subgraph that localizes hand landmarks for multiple hands (see
|
||||
# multi_hand_landmark.pbtxt).
|
||||
node {
|
||||
calculator: "MultiHandLandmarkSubgraph"
|
||||
input_stream: "IMAGE:throttled_input_video"
|
||||
input_stream: "NORM_RECTS:multi_hand_rects"
|
||||
output_stream: "LANDMARKS:multi_hand_landmarks"
|
||||
output_stream: "NORM_RECTS:multi_hand_rects_from_landmarks"
|
||||
}
|
||||
|
||||
# Caches a hand rectangle fed back from MultiHandLandmarkSubgraph, and upon the
|
||||
# arrival of the next input image sends out the cached rectangle with the
|
||||
# timestamp replaced by that of the input image, essentially generating a packet
|
||||
# that carries the previous hand rectangle. Note that upon the arrival of the
|
||||
# very first input image, an empty packet is sent out to jump start the
|
||||
# feedback loop.
|
||||
node {
|
||||
calculator: "PreviousLoopbackCalculator"
|
||||
input_stream: "MAIN:throttled_input_video"
|
||||
input_stream: "LOOP:multi_hand_rects_from_landmarks"
|
||||
input_stream_info: {
|
||||
tag_index: "LOOP"
|
||||
back_edge: true
|
||||
}
|
||||
output_stream: "PREV_LOOP:prev_multi_hand_rects_from_landmarks"
|
||||
}
|
||||
|
||||
# Performs association between NormalizedRect vector elements from previous
|
||||
# frame and those from the current frame if MultiHandDetectionSubgraph runs.
|
||||
# This calculator ensures that the output multi_hand_rects vector doesn't
|
||||
# contain overlapping regions based on the specified min_similarity_threshold.
|
||||
node {
|
||||
calculator: "AssociationNormRectCalculator"
|
||||
input_stream: "prev_multi_hand_rects_from_landmarks"
|
||||
input_stream: "multi_palm_rects"
|
||||
output_stream: "multi_hand_rects"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.AssociationCalculatorOptions] {
|
||||
min_similarity_threshold: 0.1
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Subgraph that renders annotations and overlays them on top of the input
|
||||
# images (see multi_hand_renderer_gpu.pbtxt).
|
||||
node {
|
||||
calculator: "MultiHandRendererSubgraph"
|
||||
input_stream: "IMAGE:throttled_input_video"
|
||||
input_stream: "DETECTIONS:multi_palm_detections"
|
||||
input_stream: "LANDMARKS:multi_hand_landmarks"
|
||||
input_stream: "NORM_RECTS:0:multi_palm_rects"
|
||||
input_stream: "NORM_RECTS:1:multi_hand_rects"
|
||||
output_stream: "IMAGE:output_video"
|
||||
}
|
||||
@@ -12,15 +12,15 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
licenses(["notice"]) # Apache 2.0
|
||||
|
||||
package(default_visibility = ["//visibility:public"])
|
||||
|
||||
load(
|
||||
"//mediapipe/framework/tool:mediapipe_graph.bzl",
|
||||
"mediapipe_simple_subgraph",
|
||||
)
|
||||
|
||||
licenses(["notice"]) # Apache 2.0
|
||||
|
||||
package(default_visibility = ["//visibility:public"])
|
||||
|
||||
mediapipe_simple_subgraph(
|
||||
name = "hand_detection_cpu",
|
||||
graph = "hand_detection_cpu.pbtxt",
|
||||
@@ -42,6 +42,29 @@ mediapipe_simple_subgraph(
|
||||
],
|
||||
)
|
||||
|
||||
mediapipe_simple_subgraph(
|
||||
name = "multi_hand_detection_cpu",
|
||||
graph = "multi_hand_detection_cpu.pbtxt",
|
||||
register_as = "MultiHandDetectionSubgraph",
|
||||
deps = [
|
||||
"//mediapipe/calculators/core:begin_loop_calculator",
|
||||
"//mediapipe/calculators/core:clip_vector_size_calculator",
|
||||
"//mediapipe/calculators/core:end_loop_calculator",
|
||||
"//mediapipe/calculators/image:image_properties_calculator",
|
||||
"//mediapipe/calculators/image:image_transformation_calculator",
|
||||
"//mediapipe/calculators/tflite:ssd_anchors_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_converter_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_custom_op_resolver_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_inference_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_tensors_to_detections_calculator",
|
||||
"//mediapipe/calculators/util:detection_label_id_to_text_calculator",
|
||||
"//mediapipe/calculators/util:detection_letterbox_removal_calculator",
|
||||
"//mediapipe/calculators/util:detections_to_rects_calculator",
|
||||
"//mediapipe/calculators/util:non_max_suppression_calculator",
|
||||
"//mediapipe/calculators/util:rect_transformation_calculator",
|
||||
],
|
||||
)
|
||||
|
||||
mediapipe_simple_subgraph(
|
||||
name = "hand_landmark_cpu",
|
||||
graph = "hand_landmark_cpu.pbtxt",
|
||||
@@ -65,6 +88,18 @@ mediapipe_simple_subgraph(
|
||||
],
|
||||
)
|
||||
|
||||
mediapipe_simple_subgraph(
|
||||
name = "multi_hand_landmark_cpu",
|
||||
graph = "multi_hand_landmark.pbtxt",
|
||||
register_as = "MultiHandLandmarkSubgraph",
|
||||
deps = [
|
||||
":hand_landmark_cpu",
|
||||
"//mediapipe/calculators/core:begin_loop_calculator",
|
||||
"//mediapipe/calculators/core:end_loop_calculator",
|
||||
"//mediapipe/calculators/util:filter_collection_calculator",
|
||||
],
|
||||
)
|
||||
|
||||
mediapipe_simple_subgraph(
|
||||
name = "renderer_cpu",
|
||||
graph = "renderer_cpu.pbtxt",
|
||||
@@ -77,6 +112,20 @@ mediapipe_simple_subgraph(
|
||||
],
|
||||
)
|
||||
|
||||
mediapipe_simple_subgraph(
|
||||
name = "multi_hand_renderer_cpu",
|
||||
graph = "multi_hand_renderer_cpu.pbtxt",
|
||||
register_as = "MultiHandRendererSubgraph",
|
||||
deps = [
|
||||
"//mediapipe/calculators/core:begin_loop_calculator",
|
||||
"//mediapipe/calculators/core:end_loop_calculator",
|
||||
"//mediapipe/calculators/util:annotation_overlay_calculator",
|
||||
"//mediapipe/calculators/util:detections_to_render_data_calculator",
|
||||
"//mediapipe/calculators/util:landmarks_to_render_data_calculator",
|
||||
"//mediapipe/calculators/util:rect_to_render_data_calculator",
|
||||
],
|
||||
)
|
||||
|
||||
mediapipe_simple_subgraph(
|
||||
name = "hand_detection_gpu",
|
||||
graph = "hand_detection_gpu.pbtxt",
|
||||
@@ -97,6 +146,29 @@ mediapipe_simple_subgraph(
|
||||
],
|
||||
)
|
||||
|
||||
mediapipe_simple_subgraph(
|
||||
name = "multi_hand_detection_gpu",
|
||||
graph = "multi_hand_detection_gpu.pbtxt",
|
||||
register_as = "MultiHandDetectionSubgraph",
|
||||
deps = [
|
||||
"//mediapipe/calculators/core:begin_loop_calculator",
|
||||
"//mediapipe/calculators/core:clip_vector_size_calculator",
|
||||
"//mediapipe/calculators/core:end_loop_calculator",
|
||||
"//mediapipe/calculators/image:image_properties_calculator",
|
||||
"//mediapipe/calculators/image:image_transformation_calculator",
|
||||
"//mediapipe/calculators/tflite:ssd_anchors_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_converter_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_custom_op_resolver_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_inference_calculator",
|
||||
"//mediapipe/calculators/tflite:tflite_tensors_to_detections_calculator",
|
||||
"//mediapipe/calculators/util:detection_label_id_to_text_calculator",
|
||||
"//mediapipe/calculators/util:detection_letterbox_removal_calculator",
|
||||
"//mediapipe/calculators/util:detections_to_rects_calculator",
|
||||
"//mediapipe/calculators/util:non_max_suppression_calculator",
|
||||
"//mediapipe/calculators/util:rect_transformation_calculator",
|
||||
],
|
||||
)
|
||||
|
||||
mediapipe_simple_subgraph(
|
||||
name = "hand_landmark_gpu",
|
||||
graph = "hand_landmark_gpu.pbtxt",
|
||||
@@ -119,6 +191,18 @@ mediapipe_simple_subgraph(
|
||||
],
|
||||
)
|
||||
|
||||
mediapipe_simple_subgraph(
|
||||
name = "multi_hand_landmark_gpu",
|
||||
graph = "multi_hand_landmark.pbtxt",
|
||||
register_as = "MultiHandLandmarkSubgraph",
|
||||
deps = [
|
||||
":hand_landmark_gpu",
|
||||
"//mediapipe/calculators/core:begin_loop_calculator",
|
||||
"//mediapipe/calculators/core:end_loop_calculator",
|
||||
"//mediapipe/calculators/util:filter_collection_calculator",
|
||||
],
|
||||
)
|
||||
|
||||
mediapipe_simple_subgraph(
|
||||
name = "renderer_gpu",
|
||||
graph = "renderer_gpu.pbtxt",
|
||||
@@ -130,3 +214,17 @@ mediapipe_simple_subgraph(
|
||||
"//mediapipe/calculators/util:rect_to_render_data_calculator",
|
||||
],
|
||||
)
|
||||
|
||||
mediapipe_simple_subgraph(
|
||||
name = "multi_hand_renderer_gpu",
|
||||
graph = "multi_hand_renderer_gpu.pbtxt",
|
||||
register_as = "MultiHandRendererSubgraph",
|
||||
deps = [
|
||||
"//mediapipe/calculators/core:begin_loop_calculator",
|
||||
"//mediapipe/calculators/core:end_loop_calculator",
|
||||
"//mediapipe/calculators/util:annotation_overlay_calculator",
|
||||
"//mediapipe/calculators/util:detections_to_render_data_calculator",
|
||||
"//mediapipe/calculators/util:landmarks_to_render_data_calculator",
|
||||
"//mediapipe/calculators/util:rect_to_render_data_calculator",
|
||||
],
|
||||
)
|
||||
|
||||
@@ -0,0 +1,213 @@
|
||||
# MediaPipe multi-hand detection subgraph.
|
||||
|
||||
type: "MultiHandDetectionSubgraph"
|
||||
|
||||
input_stream: "input_video"
|
||||
output_stream: "DETECTIONS:palm_detections"
|
||||
output_stream: "NORM_RECTS:clipped_hand_rects_from_palm_detections"
|
||||
|
||||
# Transforms the input image on CPU to a 256x256 image. To scale the input
|
||||
# image, the scale_mode option is set to FIT to preserve the aspect ratio,
|
||||
# resulting in potential letterboxing in the transformed image.
|
||||
node: {
|
||||
calculator: "ImageTransformationCalculator"
|
||||
input_stream: "IMAGE:input_video"
|
||||
output_stream: "IMAGE:transformed_input_video"
|
||||
output_stream: "LETTERBOX_PADDING:letterbox_padding"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.ImageTransformationCalculatorOptions] {
|
||||
output_width: 256
|
||||
output_height: 256
|
||||
scale_mode: FIT
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Generates a single side packet containing a TensorFlow Lite op resolver that
|
||||
# supports custom ops needed by the model used in this graph.
|
||||
node {
|
||||
calculator: "TfLiteCustomOpResolverCalculator"
|
||||
output_side_packet: "opresolver"
|
||||
}
|
||||
|
||||
# Converts the transformed input image on CPU into an image tensor as a
|
||||
# TfLiteTensor. The zero_center option is set to true to normalize the
|
||||
# pixel values to [-1.f, 1.f] as opposed to [0.f, 1.f].
|
||||
node {
|
||||
calculator: "TfLiteConverterCalculator"
|
||||
input_stream: "IMAGE:transformed_input_video"
|
||||
output_stream: "TENSORS:image_tensor"
|
||||
}
|
||||
|
||||
# Runs a TensorFlow Lite model on CPU that takes an image tensor and outputs a
|
||||
# vector of tensors representing, for instance, detection boxes/keypoints and
|
||||
# scores.
|
||||
node {
|
||||
calculator: "TfLiteInferenceCalculator"
|
||||
input_stream: "TENSORS:image_tensor"
|
||||
output_stream: "TENSORS:detection_tensors"
|
||||
input_side_packet: "CUSTOM_OP_RESOLVER:opresolver"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.TfLiteInferenceCalculatorOptions] {
|
||||
model_path: "mediapipe/models/palm_detection.tflite"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Generates a single side packet containing a vector of SSD anchors based on
|
||||
# the specification in the options.
|
||||
node {
|
||||
calculator: "SsdAnchorsCalculator"
|
||||
output_side_packet: "anchors"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.SsdAnchorsCalculatorOptions] {
|
||||
num_layers: 5
|
||||
min_scale: 0.1171875
|
||||
max_scale: 0.75
|
||||
input_size_height: 256
|
||||
input_size_width: 256
|
||||
anchor_offset_x: 0.5
|
||||
anchor_offset_y: 0.5
|
||||
strides: 8
|
||||
strides: 16
|
||||
strides: 32
|
||||
strides: 32
|
||||
strides: 32
|
||||
aspect_ratios: 1.0
|
||||
fixed_anchor_size: true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Decodes the detection tensors generated by the TensorFlow Lite model, based on
|
||||
# the SSD anchors and the specification in the options, into a vector of
|
||||
# detections. Each detection describes a detected object.
|
||||
node {
|
||||
calculator: "TfLiteTensorsToDetectionsCalculator"
|
||||
input_stream: "TENSORS:detection_tensors"
|
||||
input_side_packet: "ANCHORS:anchors"
|
||||
output_stream: "DETECTIONS:detections"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.TfLiteTensorsToDetectionsCalculatorOptions] {
|
||||
num_classes: 1
|
||||
num_boxes: 2944
|
||||
num_coords: 18
|
||||
box_coord_offset: 0
|
||||
keypoint_coord_offset: 4
|
||||
num_keypoints: 7
|
||||
num_values_per_keypoint: 2
|
||||
sigmoid_score: true
|
||||
score_clipping_thresh: 100.0
|
||||
reverse_output_order: true
|
||||
|
||||
x_scale: 256.0
|
||||
y_scale: 256.0
|
||||
h_scale: 256.0
|
||||
w_scale: 256.0
|
||||
min_score_thresh: 0.5
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Performs non-max suppression to remove excessive detections.
|
||||
node {
|
||||
calculator: "NonMaxSuppressionCalculator"
|
||||
input_stream: "detections"
|
||||
output_stream: "filtered_detections"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.NonMaxSuppressionCalculatorOptions] {
|
||||
min_suppression_threshold: 0.3
|
||||
min_score_threshold: 0.5
|
||||
overlap_type: INTERSECTION_OVER_UNION
|
||||
algorithm: WEIGHTED
|
||||
return_empty_detections: true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Maps detection label IDs to the corresponding label text ("Palm"). The label
|
||||
# map is provided in the label_map_path option.
|
||||
node {
|
||||
calculator: "DetectionLabelIdToTextCalculator"
|
||||
input_stream: "filtered_detections"
|
||||
output_stream: "labeled_detections"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.DetectionLabelIdToTextCalculatorOptions] {
|
||||
label_map_path: "mediapipe/models/palm_detection_labelmap.txt"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Adjusts detection locations (already normalized to [0.f, 1.f]) on the
|
||||
# letterboxed image (after image transformation with the FIT scale mode) to the
|
||||
# corresponding locations on the same image with the letterbox removed (the
|
||||
# input image to the graph before image transformation).
|
||||
node {
|
||||
calculator: "DetectionLetterboxRemovalCalculator"
|
||||
input_stream: "DETECTIONS:labeled_detections"
|
||||
input_stream: "LETTERBOX_PADDING:letterbox_padding"
|
||||
output_stream: "DETECTIONS:palm_detections"
|
||||
}
|
||||
|
||||
# Extracts image size from the input images.
|
||||
node {
|
||||
calculator: "ImagePropertiesCalculator"
|
||||
input_stream: "IMAGE:input_video"
|
||||
output_stream: "SIZE:image_size"
|
||||
}
|
||||
|
||||
# Converts each palm detection into a rectangle (normalized by image size)
|
||||
# that encloses the palm and is rotated such that the line connecting center of
|
||||
# the wrist and MCP of the middle finger is aligned with the Y-axis of the
|
||||
# rectangle.
|
||||
node {
|
||||
calculator: "DetectionsToRectsCalculator"
|
||||
input_stream: "DETECTIONS:palm_detections"
|
||||
input_stream: "IMAGE_SIZE:image_size"
|
||||
output_stream: "NORM_RECTS:palm_rects"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.DetectionsToRectsCalculatorOptions] {
|
||||
rotation_vector_start_keypoint_index: 0 # Center of wrist.
|
||||
rotation_vector_end_keypoint_index: 2 # MCP of middle finger.
|
||||
rotation_vector_target_angle_degrees: 90
|
||||
output_zero_rect_for_empty_detections: true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Expands and shifts the rectangle that contains the palm so that it's likely
|
||||
# to cover the entire hand.
|
||||
node {
|
||||
calculator: "RectTransformationCalculator"
|
||||
input_stream: "NORM_RECTS:palm_rects"
|
||||
input_stream: "IMAGE_SIZE:image_size"
|
||||
output_stream: "hand_rects_from_palm_detections"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.RectTransformationCalculatorOptions] {
|
||||
scale_x: 2.6
|
||||
scale_y: 2.6
|
||||
shift_y: -0.5
|
||||
square_long: true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Clips the size of the input vector to the provided max_vec_size. This
|
||||
# determines the maximum number of hand instances this graph outputs.
|
||||
# Note that the performance gain of clipping detections earlier in this graph is
|
||||
# minimal because NMS will minimize overlapping detections and the number of
|
||||
# detections isn't expected to exceed 5-10.
|
||||
node {
|
||||
calculator: "ClipNormalizedRectVectorSizeCalculator"
|
||||
input_stream: "hand_rects_from_palm_detections"
|
||||
output_stream: "clipped_hand_rects_from_palm_detections"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.ClipVectorSizeCalculatorOptions] {
|
||||
# This value can be changed to support tracking arbitrary number of hands.
|
||||
# Please also remember to modify min_size in
|
||||
# CollectionHsMinSizeCalculatorOptions in
|
||||
# mediapipe/graphs/hand_tracking/multi_hand_tracking_desktop.pbtxt.
|
||||
max_vec_size: 2
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,218 @@
|
||||
# MediaPipe multi-hand detection subgraph.
|
||||
|
||||
type: "MultiHandDetectionSubgraph"
|
||||
|
||||
input_stream: "input_video"
|
||||
output_stream: "DETECTIONS:palm_detections"
|
||||
output_stream: "NORM_RECTS:clipped_hand_rects_from_palm_detections"
|
||||
|
||||
# Transforms the input image on GPU to a 256x256 image. To scale the input
|
||||
# image, the scale_mode option is set to FIT to preserve the aspect ratio,
|
||||
# resulting in potential letterboxing in the transformed image.
|
||||
node: {
|
||||
calculator: "ImageTransformationCalculator"
|
||||
input_stream: "IMAGE_GPU:input_video"
|
||||
output_stream: "IMAGE_GPU:transformed_input_video"
|
||||
output_stream: "LETTERBOX_PADDING:letterbox_padding"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.ImageTransformationCalculatorOptions] {
|
||||
output_width: 256
|
||||
output_height: 256
|
||||
scale_mode: FIT
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Generates a single side packet containing a TensorFlow Lite op resolver that
|
||||
# supports custom ops needed by the model used in this graph.
|
||||
node {
|
||||
calculator: "TfLiteCustomOpResolverCalculator"
|
||||
output_side_packet: "opresolver"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.TfLiteCustomOpResolverCalculatorOptions] {
|
||||
use_gpu: true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Converts the transformed input image on GPU into an image tensor stored as a
|
||||
# TfLiteTensor.
|
||||
node {
|
||||
calculator: "TfLiteConverterCalculator"
|
||||
input_stream: "IMAGE_GPU:transformed_input_video"
|
||||
output_stream: "TENSORS_GPU:image_tensor"
|
||||
}
|
||||
|
||||
# Runs a TensorFlow Lite model on GPU that takes an image tensor and outputs a
|
||||
# vector of tensors representing, for instance, detection boxes/keypoints and
|
||||
# scores.
|
||||
node {
|
||||
calculator: "TfLiteInferenceCalculator"
|
||||
input_stream: "TENSORS_GPU:image_tensor"
|
||||
output_stream: "TENSORS_GPU:detection_tensors"
|
||||
input_side_packet: "CUSTOM_OP_RESOLVER:opresolver"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.TfLiteInferenceCalculatorOptions] {
|
||||
model_path: "mediapipe/models/palm_detection.tflite"
|
||||
use_gpu: true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Generates a single side packet containing a vector of SSD anchors based on
|
||||
# the specification in the options.
|
||||
node {
|
||||
calculator: "SsdAnchorsCalculator"
|
||||
output_side_packet: "anchors"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.SsdAnchorsCalculatorOptions] {
|
||||
num_layers: 5
|
||||
min_scale: 0.1171875
|
||||
max_scale: 0.75
|
||||
input_size_height: 256
|
||||
input_size_width: 256
|
||||
anchor_offset_x: 0.5
|
||||
anchor_offset_y: 0.5
|
||||
strides: 8
|
||||
strides: 16
|
||||
strides: 32
|
||||
strides: 32
|
||||
strides: 32
|
||||
aspect_ratios: 1.0
|
||||
fixed_anchor_size: true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Decodes the detection tensors generated by the TensorFlow Lite model, based on
|
||||
# the SSD anchors and the specification in the options, into a vector of
|
||||
# detections. Each detection describes a detected object.
|
||||
node {
|
||||
calculator: "TfLiteTensorsToDetectionsCalculator"
|
||||
input_stream: "TENSORS_GPU:detection_tensors"
|
||||
input_side_packet: "ANCHORS:anchors"
|
||||
output_stream: "DETECTIONS:detections"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.TfLiteTensorsToDetectionsCalculatorOptions] {
|
||||
num_classes: 1
|
||||
num_boxes: 2944
|
||||
num_coords: 18
|
||||
box_coord_offset: 0
|
||||
keypoint_coord_offset: 4
|
||||
num_keypoints: 7
|
||||
num_values_per_keypoint: 2
|
||||
sigmoid_score: true
|
||||
score_clipping_thresh: 100.0
|
||||
reverse_output_order: true
|
||||
|
||||
x_scale: 256.0
|
||||
y_scale: 256.0
|
||||
h_scale: 256.0
|
||||
w_scale: 256.0
|
||||
min_score_thresh: 0.7
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Performs non-max suppression to remove excessive detections.
|
||||
node {
|
||||
calculator: "NonMaxSuppressionCalculator"
|
||||
input_stream: "detections"
|
||||
output_stream: "filtered_detections"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.NonMaxSuppressionCalculatorOptions] {
|
||||
min_suppression_threshold: 0.3
|
||||
overlap_type: INTERSECTION_OVER_UNION
|
||||
algorithm: WEIGHTED
|
||||
return_empty_detections: true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Maps detection label IDs to the corresponding label text ("Palm"). The label
|
||||
# map is provided in the label_map_path option.
|
||||
node {
|
||||
calculator: "DetectionLabelIdToTextCalculator"
|
||||
input_stream: "filtered_detections"
|
||||
output_stream: "labeled_detections"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.DetectionLabelIdToTextCalculatorOptions] {
|
||||
label_map_path: "mediapipe/models/palm_detection_labelmap.txt"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Adjusts detection locations (already normalized to [0.f, 1.f]) on the
|
||||
# letterboxed image (after image transformation with the FIT scale mode) to the
|
||||
# corresponding locations on the same image with the letterbox removed (the
|
||||
# input image to the graph before image transformation).
|
||||
node {
|
||||
calculator: "DetectionLetterboxRemovalCalculator"
|
||||
input_stream: "DETECTIONS:labeled_detections"
|
||||
input_stream: "LETTERBOX_PADDING:letterbox_padding"
|
||||
output_stream: "DETECTIONS:palm_detections"
|
||||
}
|
||||
|
||||
# Extracts image size from the input images.
|
||||
node {
|
||||
calculator: "ImagePropertiesCalculator"
|
||||
input_stream: "IMAGE_GPU:input_video"
|
||||
output_stream: "SIZE:image_size"
|
||||
}
|
||||
|
||||
# Converts each palm detection into a rectangle (normalized by image size)
|
||||
# that encloses the palm and is rotated such that the line connecting center of
|
||||
# the wrist and MCP of the middle finger is aligned with the Y-axis of the
|
||||
# rectangle.
|
||||
node {
|
||||
calculator: "DetectionsToRectsCalculator"
|
||||
input_stream: "DETECTIONS:palm_detections"
|
||||
input_stream: "IMAGE_SIZE:image_size"
|
||||
output_stream: "NORM_RECTS:palm_rects"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.DetectionsToRectsCalculatorOptions] {
|
||||
rotation_vector_start_keypoint_index: 0 # Center of wrist.
|
||||
rotation_vector_end_keypoint_index: 2 # MCP of middle finger.
|
||||
rotation_vector_target_angle_degrees: 90
|
||||
output_zero_rect_for_empty_detections: true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Expands and shifts the rectangle that contains the palm so that it's likely
|
||||
# to cover the entire hand.
|
||||
node {
|
||||
calculator: "RectTransformationCalculator"
|
||||
input_stream: "NORM_RECTS:palm_rects"
|
||||
input_stream: "IMAGE_SIZE:image_size"
|
||||
output_stream: "hand_rects_from_palm_detections"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.RectTransformationCalculatorOptions] {
|
||||
scale_x: 2.6
|
||||
scale_y: 2.6
|
||||
shift_y: -0.5
|
||||
square_long: true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Clips the size of the input vector to the provided max_vec_size. This
|
||||
# determines the maximum number of hand instances this graph outputs.
|
||||
# Note that the performance gain of clipping detections earlier in this graph is
|
||||
# minimal because NMS will minimize overlapping detections and the number of
|
||||
# detections isn't expected to exceed 5-10.
|
||||
node {
|
||||
calculator: "ClipNormalizedRectVectorSizeCalculator"
|
||||
input_stream: "hand_rects_from_palm_detections"
|
||||
output_stream: "clipped_hand_rects_from_palm_detections"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.ClipVectorSizeCalculatorOptions] {
|
||||
# This value can be changed to support tracking arbitrary number of hands.
|
||||
# Please also remember to modify min_size in
|
||||
# CollectionHsMinSizeCalculatorOptions in
|
||||
# mediapipe/graphs/hand_tracking/multi_hand_tracking_mobile.pbtxt and
|
||||
# mediapipe/graphs/hand_tracking/multi_hand_tracking_desktop_live.pbtxt.
|
||||
max_vec_size: 2
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
# MediaPipe hand landmark localization subgraph.
|
||||
|
||||
type: "MultiHandLandmarkSubgraph"
|
||||
|
||||
input_stream: "IMAGE:input_video"
|
||||
# A vector of NormalizedRect, one per each hand detected.
|
||||
input_stream: "NORM_RECTS:multi_hand_rects"
|
||||
# A vector of NormalizedLandmarks, one set per each hand.
|
||||
output_stream: "LANDMARKS:filtered_multi_hand_landmarks"
|
||||
# A vector of NormalizedRect, one per each hand.
|
||||
output_stream: "NORM_RECTS:filtered_multi_hand_rects_for_next_frame"
|
||||
|
||||
# Outputs each element of multi_hand_rects at a fake timestamp for the rest
|
||||
# of the graph to process. Clones the input_video packet for each
|
||||
# single_hand_rect at the fake timestamp. At the end of the loop,
|
||||
# outputs the BATCH_END timestamp for downstream calculators to inform them
|
||||
# that all elements in the vector have been processed.
|
||||
node {
|
||||
calculator: "BeginLoopNormalizedRectCalculator"
|
||||
input_stream: "ITERABLE:multi_hand_rects"
|
||||
input_stream: "CLONE:input_video"
|
||||
output_stream: "ITEM:single_hand_rect"
|
||||
output_stream: "CLONE:input_video_cloned"
|
||||
output_stream: "BATCH_END:single_hand_rect_timestamp"
|
||||
}
|
||||
|
||||
node {
|
||||
calculator: "HandLandmarkSubgraph"
|
||||
input_stream: "IMAGE:input_video_cloned"
|
||||
input_stream: "NORM_RECT:single_hand_rect"
|
||||
output_stream: "LANDMARKS:single_hand_landmarks"
|
||||
output_stream: "NORM_RECT:single_hand_rect_from_landmarks"
|
||||
output_stream: "PRESENCE:single_hand_presence"
|
||||
}
|
||||
|
||||
# Collects the boolean presence value for each single hand into a vector. Upon
|
||||
# receiving the BATCH_END timestamp, outputs a vector of boolean values at the
|
||||
# BATCH_END timestamp.
|
||||
node {
|
||||
calculator: "EndLoopBooleanCalculator"
|
||||
input_stream: "ITEM:single_hand_presence"
|
||||
input_stream: "BATCH_END:single_hand_rect_timestamp"
|
||||
output_stream: "ITERABLE:multi_hand_presence"
|
||||
}
|
||||
|
||||
# Collects a set of landmarks for each hand into a vector. Upon receiving the
|
||||
# BATCH_END timestamp, outputs the vector of landmarks at the BATCH_END
|
||||
# timestamp.
|
||||
node {
|
||||
calculator: "EndLoopNormalizedLandmarksVectorCalculator"
|
||||
input_stream: "ITEM:single_hand_landmarks"
|
||||
input_stream: "BATCH_END:single_hand_rect_timestamp"
|
||||
output_stream: "ITERABLE:multi_hand_landmarks"
|
||||
}
|
||||
|
||||
# Collects a NormalizedRect for each hand into a vector. Upon receiving the
|
||||
# BATCH_END timestamp, outputs the vector of NormalizedRect at the BATCH_END
|
||||
# timestamp.
|
||||
node {
|
||||
calculator: "EndLoopNormalizedRectCalculator"
|
||||
input_stream: "ITEM:single_hand_rect_from_landmarks"
|
||||
input_stream: "BATCH_END:single_hand_rect_timestamp"
|
||||
output_stream: "ITERABLE:multi_hand_rects_for_next_frame"
|
||||
}
|
||||
|
||||
# Filters the input vector of landmarks based on hand presence value for each
|
||||
# hand. If the hand presence for hand #i is false, the set of landmarks
|
||||
# corresponding to that hand are dropped from the vector.
|
||||
node {
|
||||
calculator: "FilterLandmarksCollectionCalculator"
|
||||
input_stream: "ITERABLE:multi_hand_landmarks"
|
||||
input_stream: "CONDITION:multi_hand_presence"
|
||||
output_stream: "ITERABLE:filtered_multi_hand_landmarks"
|
||||
}
|
||||
|
||||
# Filters the input vector of NormalizedRect based on hand presence value for
|
||||
# each hand. If the hand presence for hand #i is false, the NormalizedRect
|
||||
# corresponding to that hand are dropped from the vector.
|
||||
node {
|
||||
calculator: "FilterNormalizedRectCollectionCalculator"
|
||||
input_stream: "ITERABLE:multi_hand_rects_for_next_frame"
|
||||
input_stream: "CONDITION:multi_hand_presence"
|
||||
output_stream: "ITERABLE:filtered_multi_hand_rects_for_next_frame"
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
# MediaPipe multi-hand tracking rendering subgraph.
|
||||
|
||||
type: "MultiHandRendererSubgraph"
|
||||
|
||||
input_stream: "IMAGE:input_image"
|
||||
# A vector of NormalizedLandmarks, one for each hand.
|
||||
input_stream: "LANDMARKS:multi_hand_landmarks"
|
||||
# A vector of NormalizedRect, one for each hand.
|
||||
input_stream: "NORM_RECTS:0:multi_palm_rects"
|
||||
# A vector of NormalizedRect, one for each hand.
|
||||
input_stream: "NORM_RECTS:1:multi_hand_rects"
|
||||
# A vector of Detection, one for each hand.
|
||||
input_stream: "DETECTIONS:palm_detections"
|
||||
output_stream: "IMAGE:output_image"
|
||||
|
||||
# Converts detections to drawing primitives for annotation overlay.
|
||||
node {
|
||||
calculator: "DetectionsToRenderDataCalculator"
|
||||
input_stream: "DETECTIONS:palm_detections"
|
||||
output_stream: "RENDER_DATA:detection_render_data"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.DetectionsToRenderDataCalculatorOptions] {
|
||||
thickness: 4.0
|
||||
color { r: 0 g: 255 b: 0 }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Converts normalized rects to drawing primitives for annotation overlay.
|
||||
node {
|
||||
calculator: "RectToRenderDataCalculator"
|
||||
input_stream: "NORM_RECTS:multi_hand_rects"
|
||||
output_stream: "RENDER_DATA:multi_hand_rects_render_data"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.RectToRenderDataCalculatorOptions] {
|
||||
filled: false
|
||||
color { r: 255 g: 0 b: 0 }
|
||||
thickness: 4.0
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Converts normalized rects to drawing primitives for annotation overlay.
|
||||
node {
|
||||
calculator: "RectToRenderDataCalculator"
|
||||
input_stream: "NORM_RECTS:multi_palm_rects"
|
||||
output_stream: "RENDER_DATA:multi_palm_rects_render_data"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.RectToRenderDataCalculatorOptions] {
|
||||
filled: false
|
||||
color { r: 125 g: 0 b: 122 }
|
||||
thickness: 4.0
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Outputs each element of multi_palm_landmarks at a fake timestamp for the rest
|
||||
# of the graph to process. At the end of the loop, outputs the BATCH_END
|
||||
# timestamp for downstream calculators to inform them that all elements in the
|
||||
# vector have been processed.
|
||||
node {
|
||||
calculator: "BeginLoopNormalizedLandmarksVectorCalculator"
|
||||
input_stream: "ITERABLE:multi_hand_landmarks"
|
||||
output_stream: "ITEM:single_hand_landmarks"
|
||||
output_stream: "BATCH_END:landmark_timestamp"
|
||||
}
|
||||
|
||||
# Converts landmarks to drawing primitives for annotation overlay.
|
||||
node {
|
||||
calculator: "LandmarksToRenderDataCalculator"
|
||||
input_stream: "NORM_LANDMARKS:single_hand_landmarks"
|
||||
output_stream: "RENDER_DATA:single_hand_landmark_render_data"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.LandmarksToRenderDataCalculatorOptions] {
|
||||
landmark_connections: 0
|
||||
landmark_connections: 1
|
||||
landmark_connections: 1
|
||||
landmark_connections: 2
|
||||
landmark_connections: 2
|
||||
landmark_connections: 3
|
||||
landmark_connections: 3
|
||||
landmark_connections: 4
|
||||
landmark_connections: 0
|
||||
landmark_connections: 5
|
||||
landmark_connections: 5
|
||||
landmark_connections: 6
|
||||
landmark_connections: 6
|
||||
landmark_connections: 7
|
||||
landmark_connections: 7
|
||||
landmark_connections: 8
|
||||
landmark_connections: 5
|
||||
landmark_connections: 9
|
||||
landmark_connections: 9
|
||||
landmark_connections: 10
|
||||
landmark_connections: 10
|
||||
landmark_connections: 11
|
||||
landmark_connections: 11
|
||||
landmark_connections: 12
|
||||
landmark_connections: 9
|
||||
landmark_connections: 13
|
||||
landmark_connections: 13
|
||||
landmark_connections: 14
|
||||
landmark_connections: 14
|
||||
landmark_connections: 15
|
||||
landmark_connections: 15
|
||||
landmark_connections: 16
|
||||
landmark_connections: 13
|
||||
landmark_connections: 17
|
||||
landmark_connections: 0
|
||||
landmark_connections: 17
|
||||
landmark_connections: 17
|
||||
landmark_connections: 18
|
||||
landmark_connections: 18
|
||||
landmark_connections: 19
|
||||
landmark_connections: 19
|
||||
landmark_connections: 20
|
||||
landmark_color { r: 255 g: 0 b: 0 }
|
||||
connection_color { r: 0 g: 255 b: 0 }
|
||||
thickness: 4.0
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Collects a RenderData object for each hand into a vector. Upon receiving the
|
||||
# BATCH_END timestamp, outputs the vector of RenderData at the BATCH_END
|
||||
# timestamp.
|
||||
node {
|
||||
calculator: "EndLoopRenderDataCalculator"
|
||||
input_stream: "ITEM:single_hand_landmark_render_data"
|
||||
input_stream: "BATCH_END:landmark_timestamp"
|
||||
output_stream: "ITERABLE:multi_hand_landmarks_render_data"
|
||||
}
|
||||
|
||||
# Draws annotations and overlays them on top of the input images. Consumes
|
||||
# a vector of RenderData objects and draws each of them on the input frame.
|
||||
node {
|
||||
calculator: "AnnotationOverlayCalculator"
|
||||
input_stream: "INPUT_FRAME:input_image"
|
||||
input_stream: "detection_render_data"
|
||||
input_stream: "multi_hand_rects_render_data"
|
||||
input_stream: "multi_palm_rects_render_data"
|
||||
input_stream: "VECTOR:0:multi_hand_landmarks_render_data"
|
||||
output_stream: "OUTPUT_FRAME:output_image"
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
# MediaPipe multi-hand tracking rendering subgraph.
|
||||
|
||||
type: "MultiHandRendererSubgraph"
|
||||
|
||||
input_stream: "IMAGE:input_image"
|
||||
# A vector of NormalizedLandmarks, one for each hand.
|
||||
input_stream: "LANDMARKS:multi_hand_landmarks"
|
||||
# A vector of NormalizedRect, one for each hand.
|
||||
input_stream: "NORM_RECTS:0:multi_palm_rects"
|
||||
# A vector of NormalizedRect, one for each hand.
|
||||
input_stream: "NORM_RECTS:1:multi_hand_rects"
|
||||
# A vector of Detection, one for each hand.
|
||||
input_stream: "DETECTIONS:palm_detections"
|
||||
output_stream: "IMAGE:output_image"
|
||||
|
||||
# Converts detections to drawing primitives for annotation overlay.
|
||||
node {
|
||||
calculator: "DetectionsToRenderDataCalculator"
|
||||
input_stream: "DETECTIONS:palm_detections"
|
||||
output_stream: "RENDER_DATA:detection_render_data"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.DetectionsToRenderDataCalculatorOptions] {
|
||||
thickness: 4.0
|
||||
color { r: 0 g: 255 b: 0 }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Converts normalized rects to drawing primitives for annotation overlay.
|
||||
node {
|
||||
calculator: "RectToRenderDataCalculator"
|
||||
input_stream: "NORM_RECTS:multi_hand_rects"
|
||||
output_stream: "RENDER_DATA:multi_hand_rects_render_data"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.RectToRenderDataCalculatorOptions] {
|
||||
filled: false
|
||||
color { r: 255 g: 0 b: 0 }
|
||||
thickness: 4.0
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Converts normalized rects to drawing primitives for annotation overlay.
|
||||
node {
|
||||
calculator: "RectToRenderDataCalculator"
|
||||
input_stream: "NORM_RECTS:multi_palm_rects"
|
||||
output_stream: "RENDER_DATA:multi_palm_rects_render_data"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.RectToRenderDataCalculatorOptions] {
|
||||
filled: false
|
||||
color { r: 125 g: 0 b: 122 }
|
||||
thickness: 4.0
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Outputs each element of multi_palm_landmarks at a fake timestamp for the rest
|
||||
# of the graph to process. At the end of the loop, outputs the BATCH_END
|
||||
# timestamp for downstream calculators to inform them that all elements in the
|
||||
# vector have been processed.
|
||||
node {
|
||||
calculator: "BeginLoopNormalizedLandmarksVectorCalculator"
|
||||
input_stream: "ITERABLE:multi_hand_landmarks"
|
||||
output_stream: "ITEM:single_hand_landmarks"
|
||||
output_stream: "BATCH_END:landmark_timestamp"
|
||||
}
|
||||
|
||||
# Converts landmarks to drawing primitives for annotation overlay.
|
||||
node {
|
||||
calculator: "LandmarksToRenderDataCalculator"
|
||||
input_stream: "NORM_LANDMARKS:single_hand_landmarks"
|
||||
output_stream: "RENDER_DATA:single_hand_landmark_render_data"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.LandmarksToRenderDataCalculatorOptions] {
|
||||
landmark_connections: 0
|
||||
landmark_connections: 1
|
||||
landmark_connections: 1
|
||||
landmark_connections: 2
|
||||
landmark_connections: 2
|
||||
landmark_connections: 3
|
||||
landmark_connections: 3
|
||||
landmark_connections: 4
|
||||
landmark_connections: 0
|
||||
landmark_connections: 5
|
||||
landmark_connections: 5
|
||||
landmark_connections: 6
|
||||
landmark_connections: 6
|
||||
landmark_connections: 7
|
||||
landmark_connections: 7
|
||||
landmark_connections: 8
|
||||
landmark_connections: 5
|
||||
landmark_connections: 9
|
||||
landmark_connections: 9
|
||||
landmark_connections: 10
|
||||
landmark_connections: 10
|
||||
landmark_connections: 11
|
||||
landmark_connections: 11
|
||||
landmark_connections: 12
|
||||
landmark_connections: 9
|
||||
landmark_connections: 13
|
||||
landmark_connections: 13
|
||||
landmark_connections: 14
|
||||
landmark_connections: 14
|
||||
landmark_connections: 15
|
||||
landmark_connections: 15
|
||||
landmark_connections: 16
|
||||
landmark_connections: 13
|
||||
landmark_connections: 17
|
||||
landmark_connections: 0
|
||||
landmark_connections: 17
|
||||
landmark_connections: 17
|
||||
landmark_connections: 18
|
||||
landmark_connections: 18
|
||||
landmark_connections: 19
|
||||
landmark_connections: 19
|
||||
landmark_connections: 20
|
||||
landmark_color { r: 255 g: 0 b: 0 }
|
||||
connection_color { r: 0 g: 255 b: 0 }
|
||||
thickness: 4.0
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Collects a RenderData object for each hand into a vector. Upon receiving the
|
||||
# BATCH_END timestamp, outputs the vector of RenderData at the BATCH_END
|
||||
# timestamp.
|
||||
node {
|
||||
calculator: "EndLoopRenderDataCalculator"
|
||||
input_stream: "ITEM:single_hand_landmark_render_data"
|
||||
input_stream: "BATCH_END:landmark_timestamp"
|
||||
output_stream: "ITERABLE:multi_hand_landmarks_render_data"
|
||||
}
|
||||
|
||||
# Draws annotations and overlays them on top of the input images. Consumes
|
||||
# a vector of RenderData objects and draws each of them on the input frame.
|
||||
node {
|
||||
calculator: "AnnotationOverlayCalculator"
|
||||
input_stream: "INPUT_FRAME_GPU:input_image"
|
||||
input_stream: "detection_render_data"
|
||||
input_stream: "multi_hand_rects_render_data"
|
||||
input_stream: "multi_palm_rects_render_data"
|
||||
input_stream: "VECTOR:0:multi_hand_landmarks_render_data"
|
||||
output_stream: "OUTPUT_FRAME_GPU:output_image"
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
# MediaPipe graph that performs object detection with TensorFlow Lite on CPU.
|
||||
# Used in the examples in
|
||||
# mediapipie/examples/desktop/object_detection:object_detection_cpu.
|
||||
# mediapipe/examples/desktop/object_detection:object_detection_cpu.
|
||||
|
||||
# Images on CPU coming into and out of the graph.
|
||||
input_stream: "input_video"
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# MediaPipe graph that performs object detection on desktop with TensorFlow Lite
|
||||
# on CPU.
|
||||
# Used in the example in
|
||||
# mediapipie/examples/desktop/object_detection:object_detection_tflite.
|
||||
# mediapipe/examples/desktop/object_detection:object_detection_tflite.
|
||||
|
||||
# max_queue_size limits the number of packets enqueued on any input stream
|
||||
# by throttling inputs to the graph. This makes the graph only process one
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# MediaPipe graph that performs object detection with TensorFlow Lite on CPU.
|
||||
# Used in the examples in
|
||||
# mediapipie/examples/android/src/java/com/mediapipe/apps/objectdetectioncpu and
|
||||
# mediapipie/examples/ios/objectdetectioncpu.
|
||||
# mediapipe/examples/android/src/java/com/mediapipe/apps/objectdetectioncpu and
|
||||
# mediapipe/examples/ios/objectdetectioncpu.
|
||||
|
||||
# Images on GPU coming into and out of the graph.
|
||||
input_stream: "input_video"
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# MediaPipe graph that performs object detection with TensorFlow Lite on GPU.
|
||||
# Used in the examples in
|
||||
# mediapipie/examples/android/src/java/com/mediapipe/apps/objectdetectiongpu and
|
||||
# mediapipie/examples/ios/objectdetectiongpu.
|
||||
# mediapipe/examples/android/src/java/com/mediapipe/apps/objectdetectiongpu and
|
||||
# mediapipe/examples/ios/objectdetectiongpu.
|
||||
|
||||
# Images on GPU coming into and out of the graph.
|
||||
input_stream: "input_video"
|
||||
|
||||
Reference in New Issue
Block a user