Project import generated by Copybara.

GitOrigin-RevId: d4a11282d20fe4d2e137f9032cf349750030dcb9

mediapipe/modules/hand_landmark/BUILD

@@ -24,7 +24,6 @@ package(default_visibility = ["//visibility:public"])
 exports_files([
     "hand_landmark_full.tflite",
     "hand_landmark_lite.tflite",
-    "hand_landmark_sparse.tflite",
     "handedness.txt",
 ])
 
@@ -56,6 +55,7 @@ mediapipe_simple_subgraph(
         "//mediapipe/calculators/util:landmark_letterbox_removal_calculator",
         "//mediapipe/calculators/util:landmark_projection_calculator",
         "//mediapipe/calculators/util:thresholding_calculator",
+        "//mediapipe/calculators/util:world_landmark_projection_calculator",
     ],
 )
 
@@ -75,6 +75,7 @@ mediapipe_simple_subgraph(
         "//mediapipe/calculators/util:landmark_letterbox_removal_calculator",
         "//mediapipe/calculators/util:landmark_projection_calculator",
         "//mediapipe/calculators/util:thresholding_calculator",
+        "//mediapipe/calculators/util:world_landmark_projection_calculator",
     ],
 )
 

mediapipe/modules/hand_landmark/hand_landmark_cpu.pbtxt

@@ -20,6 +20,16 @@ input_side_packet: "MODEL_COMPLEXITY:model_complexity"
 # the absence of this packet so that they don't wait for it unnecessarily.
 output_stream: "LANDMARKS:hand_landmarks"
 
+# Hand world landmarks within the given ROI. (LandmarkList)
+# World landmarks are real-world 3D coordinates in meters with the origin in the
+# center of the given ROI.
+#
+# WORLD_LANDMARKS shares the same landmark topology as LANDMARKS. However,
+# LANDMARKS provides coordinates (in pixels) of a 3D object projected onto the
+# 2D image surface, while WORLD_LANDMARKS provides coordinates (in meters) of
+# the 3D object itself.
+output_stream: "WORLD_LANDMARKS:hand_world_landmarks"
+
 # Handedness of the detected hand (i.e. is hand left or right).
 # (ClassificationList)
 output_stream: "HANDEDNESS:handedness"
@@ -77,11 +87,13 @@ node {
   output_stream: "landmark_tensors"
   output_stream: "hand_flag_tensor"
   output_stream: "handedness_tensor"
+  output_stream: "world_landmark_tensor"
   options: {
     [mediapipe.SplitVectorCalculatorOptions.ext] {
       ranges: { begin: 0 end: 1 }
       ranges: { begin: 1 end: 2 }
       ranges: { begin: 2 end: 3 }
+      ranges: { begin: 3 end: 4 }
     }
   }
 }
@@ -175,3 +187,33 @@ node {
   input_stream: "NORM_RECT:hand_rect"
   output_stream: "NORM_LANDMARKS:hand_landmarks"
 }
+
+# Drops world landmarks tensors if hand is not present.
+node {
+  calculator: "GateCalculator"
+  input_stream: "world_landmark_tensor"
+  input_stream: "ALLOW:hand_presence"
+  output_stream: "ensured_world_landmark_tensor"
+}
+
+# Decodes the landmark tensors into a list of landmarks, where the landmark
+# coordinates are normalized by the size of the input image to the model.
+node {
+  calculator: "TensorsToLandmarksCalculator"
+  input_stream: "TENSORS:ensured_world_landmark_tensor"
+  output_stream: "LANDMARKS:unprojected_world_landmarks"
+  options: {
+    [mediapipe.TensorsToLandmarksCalculatorOptions.ext] {
+      num_landmarks: 21
+    }
+  }
+}
+
+# Projects the world landmarks from the cropped hand image to the corresponding
+# locations on the full image before cropping (input to the graph).
+node {
+  calculator: "WorldLandmarkProjectionCalculator"
+  input_stream: "LANDMARKS:unprojected_world_landmarks"
+  input_stream: "NORM_RECT:hand_rect"
+  output_stream: "LANDMARKS:hand_world_landmarks"
+}

mediapipe/modules/hand_landmark/hand_landmark_gpu.pbtxt

@@ -20,6 +20,16 @@ input_side_packet: "MODEL_COMPLEXITY:model_complexity"
 # the absence of this packet so that they don't wait for it unnecessarily.
 output_stream: "LANDMARKS:hand_landmarks"
 
+# Hand world landmarks within the given ROI. (LandmarkList)
+# World landmarks are real-world 3D coordinates in meters with the origin in the
+# center of the given ROI.
+#
+# WORLD_LANDMARKS shares the same landmark topology as LANDMARKS. However,
+# LANDMARKS provides coordinates (in pixels) of a 3D object projected onto the
+# 2D image surface, while WORLD_LANDMARKS provides coordinates (in meters) of
+# the 3D object itself.
+output_stream: "WORLD_LANDMARKS:hand_world_landmarks"
+
 # Handedness of the detected hand (i.e. is hand left or right).
 # (ClassificationList)
 output_stream: "HANDEDNESS:handedness"
@@ -71,11 +81,13 @@ node {
   output_stream: "landmark_tensors"
   output_stream: "hand_flag_tensor"
   output_stream: "handedness_tensor"
+  output_stream: "world_landmark_tensor"
   options: {
     [mediapipe.SplitVectorCalculatorOptions.ext] {
       ranges: { begin: 0 end: 1 }
       ranges: { begin: 1 end: 2 }
       ranges: { begin: 2 end: 3 }
+      ranges: { begin: 3 end: 4 }
     }
   }
 }
@@ -169,3 +181,33 @@ node {
   input_stream: "NORM_RECT:hand_rect"
   output_stream: "NORM_LANDMARKS:hand_landmarks"
 }
+
+# Drops world landmarks tensors if hand is not present.
+node {
+  calculator: "GateCalculator"
+  input_stream: "world_landmark_tensor"
+  input_stream: "ALLOW:hand_presence"
+  output_stream: "ensured_world_landmark_tensor"
+}
+
+# Decodes the landmark tensors into a list of landmarks, where the landmark
+# coordinates are normalized by the size of the input image to the model.
+node {
+  calculator: "TensorsToLandmarksCalculator"
+  input_stream: "TENSORS:ensured_world_landmark_tensor"
+  output_stream: "LANDMARKS:unprojected_world_landmarks"
+  options: {
+    [mediapipe.TensorsToLandmarksCalculatorOptions.ext] {
+      num_landmarks: 21
+    }
+  }
+}
+
+# Projects the world landmarks from the cropped hand image to the corresponding
+# locations on the full image before cropping (input to the graph).
+node {
+  calculator: "WorldLandmarkProjectionCalculator"
+  input_stream: "LANDMARKS:unprojected_world_landmarks"
+  input_stream: "NORM_RECT:hand_rect"
+  output_stream: "LANDMARKS:hand_world_landmarks"
+}

mediapipe/modules/hand_landmark/hand_landmark_tracking_cpu.pbtxt

@@ -14,9 +14,9 @@ input_stream: "IMAGE:image"
 # Max number of hands to detect/track. (int)
 input_side_packet: "NUM_HANDS:num_hands"
 
-# Complexity of the hand landmark model: 0 or 1. Landmark accuracy as well as
-# inference latency generally go up with the model complexity. If unspecified,
-# functions as set to 1. (int)
+# Complexity of hand landmark and palm detection models: 0 or 1. Accuracy as
+# well as inference latency generally go up with the model complexity. If
+# unspecified, functions as set to 1. (int)
 input_side_packet: "MODEL_COMPLEXITY:model_complexity"
 
 # Whether landmarks on the previous image should be used to help localize
@@ -30,8 +30,22 @@ input_side_packet: "USE_PREV_LANDMARKS:use_prev_landmarks"
 # framework will internally inform the downstream calculators of the absence of
 # this packet so that they don't wait for it unnecessarily.
 output_stream: "LANDMARKS:multi_hand_landmarks"
+
+# Collection of detected/predicted hand world landmarks.
+# (std::vector<LandmarkList>)
+#
+# World landmarks are real-world 3D coordinates in meters with the origin in the
+# center of the hand bounding box calculated from the landmarks.
+#
+# WORLD_LANDMARKS shares the same landmark topology as LANDMARKS. However,
+# LANDMARKS provides coordinates (in pixels) of a 3D object projected onto the
+# 2D image surface, while WORLD_LANDMARKS provides coordinates (in meters) of
+# the 3D object itself.
+output_stream: "WORLD_LANDMARKS:multi_hand_world_landmarks"
+
 # Collection of handedness of the detected hands (i.e. is hand left or right),
-# each represented as a Classification proto.
+# each represented as a ClassificationList proto with a single Classification
+# entry. (std::vector<ClassificationList>)
 # Note that handedness is determined assuming the input image is mirrored,
 # i.e., taken with a front-facing/selfie camera with images flipped
 # horizontally.
@@ -89,6 +103,7 @@ node {
 # Detects palms.
 node {
   calculator: "PalmDetectionCpu"
+  input_side_packet: "MODEL_COMPLEXITY:model_complexity"
   input_stream: "IMAGE:palm_detection_image"
   output_stream: "DETECTIONS:all_palm_detections"
 }
@@ -186,12 +201,13 @@ node {
   input_stream: "IMAGE:image_for_landmarks"
   input_stream: "ROI:single_hand_rect"
   output_stream: "LANDMARKS:single_hand_landmarks"
+  output_stream: "WORLD_LANDMARKS:single_hand_world_landmarks"
   output_stream: "HANDEDNESS:single_handedness"
 }
 
-# Collects the handedness for each single hand into a vector. Upon
-# receiving the BATCH_END timestamp, outputs a vector of classification at the
-# BATCH_END timestamp.
+# Collects the handedness for each single hand into a vector. Upon receiving the
+# BATCH_END timestamp, outputs a vector of ClassificationList at the BATCH_END
+# timestamp.
 node {
   calculator: "EndLoopClassificationListCalculator"
   input_stream: "ITEM:single_handedness"
@@ -218,6 +234,16 @@ node {
   output_stream: "ITERABLE:multi_hand_landmarks"
 }
 
+# Collects a set of world landmarks for each hand into a vector. Upon receiving
+# the BATCH_END timestamp, outputs the vector of landmarks at the BATCH_END
+# timestamp.
+node {
+  calculator: "EndLoopLandmarkListVectorCalculator"
+  input_stream: "ITEM:single_hand_world_landmarks"
+  input_stream: "BATCH_END:hand_rects_timestamp"
+  output_stream: "ITERABLE:multi_hand_world_landmarks"
+}
+
 # Collects a NormalizedRect for each hand into a vector. Upon receiving the
 # BATCH_END timestamp, outputs the vector of NormalizedRect at the BATCH_END
 # timestamp.

mediapipe/modules/hand_landmark/hand_landmark_tracking_cpu_image.pbtxt

@@ -14,9 +14,9 @@ input_stream: "IMAGE:image"
 # Max number of hands to detect/track. (int)
 input_side_packet: "NUM_HANDS:num_hands"
 
-# Complexity of the hand landmark model: 0 or 1. Landmark accuracy as well as
-# inference latency generally go up with the model complexity. If unspecified,
-# functions as set to 1. (int)
+# Complexity of hand landmark and palm detection models: 0 or 1. Accuracy as
+# well as inference latency generally go up with the model complexity. If
+# unspecified, functions as set to 1. (int)
 input_side_packet: "MODEL_COMPLEXITY:model_complexity"
 
 # Whether landmarks on the previous image should be used to help localize
@@ -25,6 +25,7 @@ input_side_packet: "USE_PREV_LANDMARKS:use_prev_landmarks"
 
 # The throttled input image. (Image)
 output_stream: "IMAGE:throttled_image"
+
 # Collection of detected/predicted hands, each represented as a list of
 # landmarks. (std::vector<NormalizedLandmarkList>)
 # NOTE: there will not be an output packet in the LANDMARKS stream for this
@@ -32,8 +33,22 @@ output_stream: "IMAGE:throttled_image"
 # framework will internally inform the downstream calculators of the absence of
 # this packet so that they don't wait for it unnecessarily.
 output_stream: "LANDMARKS:multi_hand_landmarks"
+
+# Collection of detected/predicted hand world landmarks.
+# (std::vector<LandmarkList>)
+#
+# World landmarks are real-world 3D coordinates in meters with the origin in the
+# center of the hand bounding box calculated from the landmarks.
+#
+# WORLD_LANDMARKS shares the same landmark topology as LANDMARKS. However,
+# LANDMARKS provides coordinates (in pixels) of a 3D object projected onto the
+# 2D image surface, while WORLD_LANDMARKS provides coordinates (in meters) of
+# the 3D object itself.
+output_stream: "WORLD_LANDMARKS:multi_hand_world_landmarks"
+
 # Collection of handedness of the detected hands (i.e. is hand left or right),
-# each represented as a Classification proto.
+# each represented as a ClassificationList proto with a single Classification
+# entry. (std::vector<ClassificationList>)
 # Note that handedness is determined assuming the input image is mirrored,
 # i.e., taken with a front-facing/selfie camera with images flipped
 # horizontally.
@@ -93,6 +108,7 @@ node {
   input_side_packet: "MODEL_COMPLEXITY:model_complexity"
   input_side_packet: "USE_PREV_LANDMARKS:use_prev_landmarks"
   output_stream: "LANDMARKS:multi_hand_landmarks"
+  output_stream: "WORLD_LANDMARKS:multi_hand_world_landmarks"
   output_stream: "HANDEDNESS:multi_handedness"
   output_stream: "PALM_DETECTIONS:palm_detections"
   output_stream: "HAND_ROIS_FROM_LANDMARKS:hand_rects"

mediapipe/modules/hand_landmark/hand_landmark_tracking_gpu.pbtxt

@@ -14,9 +14,9 @@ input_stream: "IMAGE:image"
 # Max number of hands to detect/track. (int)
 input_side_packet: "NUM_HANDS:num_hands"
 
-# Complexity of the hand landmark model: 0 or 1. Landmark accuracy as well as
-# inference latency generally go up with the model complexity. If unspecified,
-# functions as set to 1. (int)
+# Complexity of hand landmark and palm detection models: 0 or 1. Accuracy as
+# well as inference latency generally go up with the model complexity. If
+# unspecified, functions as set to 1. (int)
 input_side_packet: "MODEL_COMPLEXITY:model_complexity"
 
 # Whether landmarks on the previous image should be used to help localize
@@ -30,8 +30,22 @@ input_side_packet: "USE_PREV_LANDMARKS:use_prev_landmarks"
 # framework will internally inform the downstream calculators of the absence of
 # this packet so that they don't wait for it unnecessarily.
 output_stream: "LANDMARKS:multi_hand_landmarks"
+
+# Collection of detected/predicted hand world landmarks.
+# (std::vector<LandmarkList>)
+#
+# World landmarks are real-world 3D coordinates in meters with the origin in the
+# center of the hand bounding box calculated from the landmarks.
+#
+# WORLD_LANDMARKS shares the same landmark topology as LANDMARKS. However,
+# LANDMARKS provides coordinates (in pixels) of a 3D object projected onto the
+# 2D image surface, while WORLD_LANDMARKS provides coordinates (in meters) of
+# the 3D object itself.
+output_stream: "WORLD_LANDMARKS:multi_hand_world_landmarks"
+
 # Collection of handedness of the detected hands (i.e. is hand left or right),
-# each represented as a Classification proto.
+# each represented as a ClassificationList proto with a single Classification
+# entry. (std::vector<ClassificationList>)
 # Note that handedness is determined assuming the input image is mirrored,
 # i.e., taken with a front-facing/selfie camera with images flipped
 # horizontally.
@@ -89,6 +103,7 @@ node {
 # Detects palms.
 node {
   calculator: "PalmDetectionGpu"
+  input_side_packet: "MODEL_COMPLEXITY:model_complexity"
   input_stream: "IMAGE:palm_detection_image"
   output_stream: "DETECTIONS:all_palm_detections"
 }
@@ -187,12 +202,13 @@ node {
   input_stream: "IMAGE:image_for_landmarks"
   input_stream: "ROI:single_hand_rect"
   output_stream: "LANDMARKS:single_hand_landmarks"
+  output_stream: "WORLD_LANDMARKS:single_hand_world_landmarks"
   output_stream: "HANDEDNESS:single_handedness"
 }
 
-# Collects the handedness for each single hand into a vector. Upon
-# receiving the BATCH_END timestamp, outputs a vector of classification at the
-# BATCH_END timestamp.
+# Collects the handedness for each single hand into a vector. Upon receiving the
+# BATCH_END timestamp, outputs a vector of ClassificationList at the BATCH_END
+# timestamp.
 node {
   calculator: "EndLoopClassificationListCalculator"
   input_stream: "ITEM:single_handedness"
@@ -219,6 +235,16 @@ node {
   output_stream: "ITERABLE:multi_hand_landmarks"
 }
 
+# Collects a set of world landmarks for each hand into a vector. Upon receiving
+# the BATCH_END timestamp, outputs the vector of landmarks at the BATCH_END
+# timestamp.
+node {
+  calculator: "EndLoopLandmarkListVectorCalculator"
+  input_stream: "ITEM:single_hand_world_landmarks"
+  input_stream: "BATCH_END:hand_rects_timestamp"
+  output_stream: "ITERABLE:multi_hand_world_landmarks"
+}
+
 # Collects a NormalizedRect for each hand into a vector. Upon receiving the
 # BATCH_END timestamp, outputs the vector of NormalizedRect at the BATCH_END
 # timestamp.

mediapipe/modules/hand_landmark/hand_landmark_tracking_gpu_image.pbtxt

@@ -14,9 +14,9 @@ input_stream: "IMAGE:image"
 # Max number of hands to detect/track. (int)
 input_side_packet: "NUM_HANDS:num_hands"
 
-# Complexity of the hand landmark model: 0 or 1. Landmark accuracy as well as
-# inference latency generally go up with the model complexity. If unspecified,
-# functions as set to 1. (int)
+# Complexity of hand landmark and palm detection models: 0 or 1. Accuracy as
+# well as inference latency generally go up with the model complexity. If
+# unspecified, functions as set to 1. (int)
 input_side_packet: "MODEL_COMPLEXITY:model_complexity"
 
 # Whether landmarks on the previous image should be used to help localize
@@ -30,8 +30,22 @@ input_side_packet: "USE_PREV_LANDMARKS:use_prev_landmarks"
 # framework will internally inform the downstream calculators of the absence of
 # this packet so that they don't wait for it unnecessarily.
 output_stream: "LANDMARKS:multi_hand_landmarks"
+
+# Collection of detected/predicted hand world landmarks.
+# (std::vector<LandmarkList>)
+#
+# World landmarks are real-world 3D coordinates in meters with the origin in the
+# center of the hand bounding box calculated from the landmarks.
+#
+# WORLD_LANDMARKS shares the same landmark topology as LANDMARKS. However,
+# LANDMARKS provides coordinates (in pixels) of a 3D object projected onto the
+# 2D image surface, while WORLD_LANDMARKS provides coordinates (in meters) of
+# the 3D object itself.
+output_stream: "WORLD_LANDMARKS:multi_hand_world_landmarks"
+
 # Collection of handedness of the detected hands (i.e. is hand left or right),
-# each represented as a Classification proto.
+# each represented as a ClassificationList proto with a single Classification
+# entry. (std::vector<ClassificationList>)
 # Note that handedness is determined assuming the input image is mirrored,
 # i.e., taken with a front-facing/selfie camera with images flipped
 # horizontally.
@@ -93,6 +107,7 @@ node {
   input_side_packet: "MODEL_COMPLEXITY:model_complexity"
   input_side_packet: "USE_PREV_LANDMARKS:use_prev_landmarks"
   output_stream: "LANDMARKS:multi_hand_landmarks"
+  output_stream: "WORLD_LANDMARKS:multi_hand_world_landmarks"
   output_stream: "HANDEDNESS:multi_handedness"
   output_stream: "PALM_DETECTIONS:palm_detections"
   output_stream: "HAND_ROIS_FROM_LANDMARKS:hand_rects"

mediapipe/modules/holistic_landmark/face_landmarks_from_pose_cpu.pbtxt

@@ -8,6 +8,11 @@ input_stream: "IMAGE:input_video"
 # Face-related pose landmarks. (NormalizedLandmarkList)
 input_stream: "FACE_LANDMARKS_FROM_POSE:face_landmarks_from_pose"
 
+# Whether to run the face landmark model with attention on lips and eyes to
+# provide more accuracy, and additionally output iris landmarks. If unspecified,
+# functions as set to false. (bool)
+input_side_packet: "REFINE_LANDMARKS:refine_landmarks"
+
 # Face landmarks. (NormalizedLandmarkList)
 output_stream: "FACE_LANDMARKS:face_landmarks"
 
@@ -72,5 +77,6 @@ node {
   calculator: "FaceLandmarkCpu"
   input_stream: "IMAGE:input_video"
   input_stream: "ROI:face_tracking_roi"
+  input_side_packet: "WITH_ATTENTION:refine_landmarks"
   output_stream: "LANDMARKS:face_landmarks"
 }

mediapipe/modules/holistic_landmark/face_landmarks_from_pose_gpu.pbtxt

@@ -8,6 +8,11 @@ input_stream: "IMAGE:input_video"
 # Face-related pose landmarks. (NormalizedLandmarkList)
 input_stream: "FACE_LANDMARKS_FROM_POSE:face_landmarks_from_pose"
 
+# Whether to run the face landmark model with attention on lips and eyes to
+# provide more accuracy, and additionally output iris landmarks. If unspecified,
+# functions as set to false. (bool)
+input_side_packet: "REFINE_LANDMARKS:refine_landmarks"
+
 # Face landmarks. (NormalizedLandmarkList)
 output_stream: "FACE_LANDMARKS:face_landmarks"
 
@@ -72,5 +77,6 @@ node {
   calculator: "FaceLandmarkGpu"
   input_stream: "IMAGE:input_video"
   input_stream: "ROI:face_tracking_roi"
+  input_side_packet: "WITH_ATTENTION:refine_landmarks"
   output_stream: "LANDMARKS:face_landmarks"
 }

mediapipe/modules/holistic_landmark/holistic_landmark_cpu.pbtxt

@@ -35,6 +35,7 @@
 #     input_side_packet: "SMOOTH_LANDMARKS:smooth_landmarks"
 #     input_side_packet: "ENABLE_SEGMENTATION:enable_segmentation"
 #     input_side_packet: "SMOOTH_SEGMENTATION:smooth_segmentation"
+#     input_side_packet: "REFINE_FACE_LANDMARKS:refine_face_landmarks"
 #     input_side_packet: "USE_PREV_LANDMARKS:use_prev_landmarks"
 #     output_stream: "POSE_LANDMARKS:pose_landmarks"
 #     output_stream: "FACE_LANDMARKS:face_landmarks"
@@ -70,6 +71,11 @@ input_side_packet: "ENABLE_SEGMENTATION:enable_segmentation"
 # jitter. If unspecified, functions as set to true. (bool)
 input_side_packet: "SMOOTH_SEGMENTATION:smooth_segmentation"
 
+# Whether to run the face landmark model with attention on lips and eyes to
+# provide more accuracy, and additionally output iris landmarks. If unspecified,
+# functions as set to false. (bool)
+input_side_packet: "REFINE_FACE_LANDMARKS:refine_face_landmarks"
+
 # Whether landmarks on the previous image should be used to help localize
 # landmarks on the current image. (bool)
 input_side_packet: "USE_PREV_LANDMARKS:use_prev_landmarks"
@@ -135,5 +141,6 @@ node {
   calculator: "FaceLandmarksFromPoseCpu"
   input_stream: "IMAGE:image"
   input_stream: "FACE_LANDMARKS_FROM_POSE:face_landmarks_from_pose"
+  input_side_packet: "REFINE_LANDMARKS:refine_face_landmarks"
   output_stream: "FACE_LANDMARKS:face_landmarks"
 }

mediapipe/modules/holistic_landmark/holistic_landmark_gpu.pbtxt

@@ -35,6 +35,7 @@
 #     input_side_packet: "SMOOTH_LANDMARKS:smooth_landmarks"
 #     input_side_packet: "ENABLE_SEGMENTATION:enable_segmentation"
 #     input_side_packet: "SMOOTH_SEGMENTATION:smooth_segmentation"
+#     input_side_packet: "REFINE_FACE_LANDMARKS:refine_face_landmarks"
 #     input_side_packet: "USE_PREV_LANDMARKS:use_prev_landmarks"
 #     output_stream: "POSE_LANDMARKS:pose_landmarks"
 #     output_stream: "FACE_LANDMARKS:face_landmarks"
@@ -70,6 +71,11 @@ input_side_packet: "ENABLE_SEGMENTATION:enable_segmentation"
 # jitter. If unspecified, functions as set to true. (bool)
 input_side_packet: "SMOOTH_SEGMENTATION:smooth_segmentation"
 
+# Whether to run the face landmark model with attention on lips and eyes to
+# provide more accuracy, and additionally output iris landmarks. If unspecified,
+# functions as set to false. (bool)
+input_side_packet: "REFINE_FACE_LANDMARKS:refine_face_landmarks"
+
 # Whether landmarks on the previous image should be used to help localize
 # landmarks on the current image. (bool)
 input_side_packet: "USE_PREV_LANDMARKS:use_prev_landmarks"
@@ -135,5 +141,6 @@ node {
   calculator: "FaceLandmarksFromPoseGpu"
   input_stream: "IMAGE:image"
   input_stream: "FACE_LANDMARKS_FROM_POSE:face_landmarks_from_pose"
+  input_side_packet: "REFINE_LANDMARKS:refine_face_landmarks"
   output_stream: "FACE_LANDMARKS:face_landmarks"
 }

mediapipe/modules/palm_detection/BUILD

@@ -21,13 +21,29 @@ licenses(["notice"])
 
 package(default_visibility = ["//visibility:public"])
 
-exports_files(["palm_detection.tflite"])
+exports_files([
+    "palm_detection_lite.tflite",
+    "palm_detection_full.tflite",
+])
+
+mediapipe_simple_subgraph(
+    name = "palm_detection_model_loader",
+    graph = "palm_detection_model_loader.pbtxt",
+    register_as = "PalmDetectionModelLoader",
+    deps = [
+        "//mediapipe/calculators/core:constant_side_packet_calculator",
+        "//mediapipe/calculators/tflite:tflite_model_calculator",
+        "//mediapipe/calculators/util:local_file_contents_calculator",
+        "//mediapipe/framework/tool:switch_container",
+    ],
+)
 
 mediapipe_simple_subgraph(
     name = "palm_detection_cpu",
     graph = "palm_detection_cpu.pbtxt",
     register_as = "PalmDetectionCpu",
     deps = [
+        ":palm_detection_model_loader",
         "//mediapipe/calculators/tensor:image_to_tensor_calculator",
         "//mediapipe/calculators/tensor:inference_calculator",
         "//mediapipe/calculators/tensor:tensors_to_detections_calculator",
@@ -43,6 +59,7 @@ mediapipe_simple_subgraph(
     graph = "palm_detection_gpu.pbtxt",
     register_as = "PalmDetectionGpu",
     deps = [
+        ":palm_detection_model_loader",
         "//mediapipe/calculators/tensor:image_to_tensor_calculator",
         "//mediapipe/calculators/tensor:inference_calculator",
         "//mediapipe/calculators/tensor:tensors_to_detections_calculator",
@@ -52,10 +69,3 @@ mediapipe_simple_subgraph(
         "//mediapipe/calculators/util:non_max_suppression_calculator",
     ],
 )
-
-exports_files(
-    srcs = [
-        "palm_detection.tflite",
-        "palm_detection_labelmap.txt",
-    ],
-)

mediapipe/modules/palm_detection/palm_detection_cpu.pbtxt

@@ -5,6 +5,11 @@ type: "PalmDetectionCpu"
 # CPU image. (ImageFrame)
 input_stream: "IMAGE:image"
 
+# Complexity of the palm detection model: 0 or 1. Accuracy as well as inference
+# latency generally go up with the model complexity. If unspecified, functions
+# as set to 1. (int)
+input_side_packet: "MODEL_COMPLEXITY:model_complexity"
+
 # Detected palms. (std::vector<Detection>)
 # NOTE: there will not be an output packet in the DETECTIONS stream for this
 # particular timestamp if none of palms detected. However, the MediaPipe
@@ -21,11 +26,11 @@ node {
   output_stream: "LETTERBOX_PADDING:letterbox_padding"
   options: {
     [mediapipe.ImageToTensorCalculatorOptions.ext] {
-      output_tensor_width: 128
-      output_tensor_height: 128
+      output_tensor_width: 192
+      output_tensor_height: 192
       keep_aspect_ratio: true
       output_tensor_float_range {
-        min: -1.0
+        min: 0.0
         max: 1.0
       }
       border_mode: BORDER_ZERO
@@ -39,6 +44,13 @@ node {
   output_side_packet: "opresolver"
 }
 
+# Loads the palm detection TF Lite model.
+node {
+  calculator: "PalmDetectionModelLoader"
+  input_side_packet: "MODEL_COMPLEXITY:model_complexity"
+  output_side_packet: "MODEL:model"
+}
+
 # Runs a TensorFlow Lite model on CPU that takes an image tensor and outputs a
 # vector of tensors representing, for instance, detection boxes/keypoints and
 # scores.
@@ -47,9 +59,9 @@ node {
   input_stream: "TENSORS:input_tensor"
   output_stream: "TENSORS:detection_tensors"
   input_side_packet: "CUSTOM_OP_RESOLVER:opresolver"
+  input_side_packet: "MODEL:model"
   options: {
     [mediapipe.InferenceCalculatorOptions.ext] {
-      model_path: "mediapipe/modules/palm_detection/palm_detection.tflite"
       delegate { xnnpack {} }
     }
   }
@@ -65,8 +77,8 @@ node {
       num_layers: 4
       min_scale: 0.1484375
       max_scale: 0.75
-      input_size_height: 128
-      input_size_width: 128
+      input_size_width: 192
+      input_size_height: 192
       anchor_offset_x: 0.5
       anchor_offset_y: 0.5
       strides: 8
@@ -90,7 +102,7 @@ node {
   options: {
     [mediapipe.TensorsToDetectionsCalculatorOptions.ext] {
       num_classes: 1
-      num_boxes: 896
+      num_boxes: 2016
       num_coords: 18
       box_coord_offset: 0
       keypoint_coord_offset: 4
@@ -100,10 +112,10 @@ node {
       score_clipping_thresh: 100.0
       reverse_output_order: true
 
-      x_scale: 128.0
-      y_scale: 128.0
-      h_scale: 128.0
-      w_scale: 128.0
+      x_scale: 192.0
+      y_scale: 192.0
+      w_scale: 192.0
+      h_scale: 192.0
       min_score_thresh: 0.5
     }
   }

mediapipe/modules/palm_detection/palm_detection_gpu.pbtxt

@@ -5,6 +5,11 @@ type: "PalmDetectionGpu"
 # GPU image. (GpuBuffer)
 input_stream: "IMAGE:image"
 
+# Complexity of the palm detection model: 0 or 1. Accuracy as well as inference
+# latency generally go up with the model complexity. If unspecified, functions
+# as set to 1. (int)
+input_side_packet: "MODEL_COMPLEXITY:model_complexity"
+
 # Detected palms. (std::vector<Detection>)
 # NOTE: there will not be an output packet in the DETECTIONS stream for this
 # particular timestamp if none of palms detected. However, the MediaPipe
@@ -21,11 +26,11 @@ node {
   output_stream: "LETTERBOX_PADDING:letterbox_padding"
   options: {
     [mediapipe.ImageToTensorCalculatorOptions.ext] {
-      output_tensor_width: 128
-      output_tensor_height: 128
+      output_tensor_width: 192
+      output_tensor_height: 192
       keep_aspect_ratio: true
       output_tensor_float_range {
-        min: -1.0
+        min: 0.0
         max: 1.0
       }
       border_mode: BORDER_ZERO
@@ -45,6 +50,13 @@ node {
   }
 }
 
+# Loads the palm detection TF Lite model.
+node {
+  calculator: "PalmDetectionModelLoader"
+  input_side_packet: "MODEL_COMPLEXITY:model_complexity"
+  output_side_packet: "MODEL:model"
+}
+
 # Runs a TensorFlow Lite model on GPU that takes an image tensor and outputs a
 # vector of tensors representing, for instance, detection boxes/keypoints and
 # scores.
@@ -53,10 +65,10 @@ node {
   input_stream: "TENSORS:input_tensor"
   output_stream: "TENSORS:detection_tensors"
   input_side_packet: "CUSTOM_OP_RESOLVER:opresolver"
+  input_side_packet: "MODEL:model"
   options: {
     [mediapipe.InferenceCalculatorOptions.ext] {
-      model_path: "mediapipe/modules/palm_detection/palm_detection.tflite"
-      use_gpu: true
+      delegate { gpu {} }
     }
   }
 }
@@ -71,8 +83,8 @@ node {
       num_layers: 4
       min_scale: 0.1484375
       max_scale: 0.75
-      input_size_height: 128
-      input_size_width: 128
+      input_size_width: 192
+      input_size_height: 192
       anchor_offset_x: 0.5
       anchor_offset_y: 0.5
       strides: 8
@@ -96,7 +108,7 @@ node {
   options: {
     [mediapipe.TensorsToDetectionsCalculatorOptions.ext] {
       num_classes: 1
-      num_boxes: 896
+      num_boxes: 2016
       num_coords: 18
       box_coord_offset: 0
       keypoint_coord_offset: 4
@@ -106,10 +118,10 @@ node {
       score_clipping_thresh: 100.0
       reverse_output_order: true
 
-      x_scale: 128.0
-      y_scale: 128.0
-      h_scale: 128.0
-      w_scale: 128.0
+      x_scale: 192.0
+      y_scale: 192.0
+      w_scale: 192.0
+      h_scale: 192.0
       min_score_thresh: 0.5
     }
   }

mediapipe/modules/palm_detection/palm_detection_model_loader.pbtxt

@@ -0,0 +1,63 @@
+# MediaPipe graph to load a selected palm detection TF Lite model.
+
+type: "PalmDetectionModelLoader"
+
+# Complexity of the palm detection model: 0 or 1. Accuracy as well as inference
+# latency generally go up with the model complexity. If unspecified, functions
+# as set to 1. (int)
+input_side_packet: "MODEL_COMPLEXITY:model_complexity"
+
+# TF Lite model represented as a FlatBuffer.
+# (std::unique_ptr<tflite::FlatBufferModel, std::function<void(tflite::FlatBufferModel*)>>)
+output_side_packet: "MODEL:model"
+
+# Determines path to the desired pose landmark model file.
+node {
+  calculator: "SwitchContainer"
+  input_side_packet: "SELECT:model_complexity"
+  output_side_packet: "PACKET:model_path"
+  options: {
+    [mediapipe.SwitchContainerOptions.ext] {
+      select: 1
+      contained_node: {
+        calculator: "ConstantSidePacketCalculator"
+        options: {
+          [mediapipe.ConstantSidePacketCalculatorOptions.ext]: {
+            packet {
+              string_value: "mediapipe/modules/palm_detection/palm_detection_lite.tflite"
+            }
+          }
+        }
+      }
+      contained_node: {
+        calculator: "ConstantSidePacketCalculator"
+        options: {
+          [mediapipe.ConstantSidePacketCalculatorOptions.ext]: {
+            packet {
+              string_value: "mediapipe/modules/palm_detection/palm_detection_full.tflite"
+            }
+          }
+        }
+      }
+    }
+  }
+}
+
+# Loads the file in the specified path into a blob.
+node {
+  calculator: "LocalFileContentsCalculator"
+  input_side_packet: "FILE_PATH:model_path"
+  output_side_packet: "CONTENTS:model_blob"
+  options: {
+    [mediapipe.LocalFileContentsCalculatorOptions.ext]: {
+      text_mode: false
+    }
+  }
+}
+
+# Converts the input blob into a TF Lite model.
+node {
+  calculator: "TfLiteModelCalculator"
+  input_side_packet: "MODEL_BLOB:model_blob"
+  output_side_packet: "MODEL:model"
+}