Project import generated by Copybara.

GitOrigin-RevId: d8caa66de45839696f5bd0786ad3bfbcb9cff632

mediapipe/modules/README.md

@@ -10,7 +10,9 @@ Each module (represented as a subfolder) provides subgraphs and corresponding re
 | [`face_geometry`](face_geometry/README.md) | Subgraphs to extract face geometry. |
 | [`face_landmark`](face_landmark/README.md) | Subgraphs to detect and track face landmarks. |
 | [`hand_landmark`](hand_landmark/README.md) | Subgraphs to detect and track hand landmarks. |
+| [`holistic_landmark`](holistic_landmark/README.md) | Subgraphs to detect and track holistic pose which consists of pose, face and hand landmarks. |
 | [`iris_landmark`](iris_landmark/README.md) | Subgraphs to detect iris landmarks. |
 | [`palm_detection`](palm_detection/README.md) | Subgraphs to detect palms/hands. |
 | [`pose_detection`](pose_detection/README.md) | Subgraphs to detect poses. |
 | [`pose_landmark`](pose_landmark/README.md) | Subgraphs to detect and track pose landmarks. |
+| [`objectron`](objectron/README.md) | Subgraphs to detect and track 3D objects. |

mediapipe/modules/face_detection/BUILD

@@ -21,6 +21,34 @@ licenses(["notice"])
 
 package(default_visibility = ["//visibility:public"])
 
+mediapipe_simple_subgraph(
+    name = "face_detection_front_by_roi_cpu",
+    graph = "face_detection_front_by_roi_cpu.pbtxt",
+    register_as = "FaceDetectionFrontByRoiCpu",
+    deps = [
+        "//mediapipe/calculators/tensor:image_to_tensor_calculator",
+        "//mediapipe/calculators/tensor:inference_calculator",
+        "//mediapipe/calculators/tensor:tensors_to_detections_calculator",
+        "//mediapipe/calculators/tflite:ssd_anchors_calculator",
+        "//mediapipe/calculators/util:detection_projection_calculator",
+        "//mediapipe/calculators/util:non_max_suppression_calculator",
+    ],
+)
+
+mediapipe_simple_subgraph(
+    name = "face_detection_front_by_roi_gpu",
+    graph = "face_detection_front_by_roi_gpu.pbtxt",
+    register_as = "FaceDetectionFrontByRoiGpu",
+    deps = [
+        "//mediapipe/calculators/tensor:image_to_tensor_calculator",
+        "//mediapipe/calculators/tensor:inference_calculator",
+        "//mediapipe/calculators/tensor:tensors_to_detections_calculator",
+        "//mediapipe/calculators/tflite:ssd_anchors_calculator",
+        "//mediapipe/calculators/util:detection_projection_calculator",
+        "//mediapipe/calculators/util:non_max_suppression_calculator",
+    ],
+)
+
 mediapipe_simple_subgraph(
     name = "face_detection_front_cpu",
     graph = "face_detection_front_cpu.pbtxt",

mediapipe/modules/face_detection/face_detection_front_by_roi_cpu.pbtxt

@@ -0,0 +1,143 @@
+# MediaPipe graph to detect faces. (CPU input, and inference is executed on
+# CPU.)
+#
+# It is required that "face_detection_front.tflite" is available at
+# "mediapipe/modules/face_detection/face_detection_front.tflite"
+# path during execution.
+#
+# EXAMPLE:
+#   node {
+#     calculator: "FaceDetectionFrontByRoiCpu"
+#     input_stream: "IMAGE:image"
+#     input_stream: "ROI:roi"
+#     output_stream: "DETECTIONS:face_detections"
+#   }
+
+type: "FaceDetectionFrontCpu"
+
+# CPU image. (ImageFrame)
+input_stream: "IMAGE:image"
+
+# ROI (region of interest) within the given image where faces should be
+# detected. (NormalizedRect)
+input_stream: "ROI:roi"
+
+# Detected faces. (std::vector<Detection>)
+# NOTE: there will not be an output packet in the DETECTIONS stream for this
+# particular timestamp if none of faces detected. However, the MediaPipe
+# framework will internally inform the downstream calculators of the absence of
+# this packet so that they don't wait for it unnecessarily.
+output_stream: "DETECTIONS:detections"
+
+# Transforms specified region of image into 128x128 tensor keeping aspect ratio
+# (padding tensor if needed).
+node {
+  calculator: "ImageToTensorCalculator"
+  input_stream: "IMAGE:image"
+  input_stream: "NORM_RECT:roi"
+  output_stream: "TENSORS:input_tensors"
+  output_stream: "MATRIX:transform_matrix"
+  options: {
+    [mediapipe.ImageToTensorCalculatorOptions.ext] {
+      output_tensor_width: 128
+      output_tensor_height: 128
+      keep_aspect_ratio: true
+      output_tensor_float_range {
+        min: -1.0
+        max: 1.0
+      }
+      border_mode: BORDER_ZERO
+    }
+  }
+}
+
+# Runs a TensorFlow Lite model on CPU that takes an image tensor and outputs a
+# vector of tensors representing, for instance, detection boxes/keypoints and
+# scores.
+node {
+  calculator: "InferenceCalculator"
+  input_stream: "TENSORS:input_tensors"
+  output_stream: "TENSORS:detection_tensors"
+  options: {
+    [mediapipe.InferenceCalculatorOptions.ext] {
+      model_path: "mediapipe/modules/face_detection/face_detection_front.tflite"
+      delegate { xnnpack {} }
+    }
+  }
+}
+
+# Generates a single side packet containing a vector of SSD anchors based on
+# the specification in the options.
+node {
+  calculator: "SsdAnchorsCalculator"
+  output_side_packet: "anchors"
+  options: {
+    [mediapipe.SsdAnchorsCalculatorOptions.ext] {
+      num_layers: 4
+      min_scale: 0.1484375
+      max_scale: 0.75
+      input_size_height: 128
+      input_size_width: 128
+      anchor_offset_x: 0.5
+      anchor_offset_y: 0.5
+      strides: 8
+      strides: 16
+      strides: 16
+      strides: 16
+      aspect_ratios: 1.0
+      fixed_anchor_size: true
+    }
+  }
+}
+
+# Decodes the detection tensors generated by the TensorFlow Lite model, based on
+# the SSD anchors and the specification in the options, into a vector of
+# detections. Each detection describes a detected object.
+node {
+  calculator: "TensorsToDetectionsCalculator"
+  input_stream: "TENSORS:detection_tensors"
+  input_side_packet: "ANCHORS:anchors"
+  output_stream: "DETECTIONS:unfiltered_detections"
+  options: {
+    [mediapipe.TensorsToDetectionsCalculatorOptions.ext] {
+      num_classes: 1
+      num_boxes: 896
+      num_coords: 16
+      box_coord_offset: 0
+      keypoint_coord_offset: 4
+      num_keypoints: 6
+      num_values_per_keypoint: 2
+      sigmoid_score: true
+      score_clipping_thresh: 100.0
+      reverse_output_order: true
+      x_scale: 128.0
+      y_scale: 128.0
+      h_scale: 128.0
+      w_scale: 128.0
+      min_score_thresh: 0.5
+    }
+  }
+}
+
+# Performs non-max suppression to remove excessive detections.
+node {
+  calculator: "NonMaxSuppressionCalculator"
+  input_stream: "unfiltered_detections"
+  output_stream: "filtered_detections"
+  options: {
+    [mediapipe.NonMaxSuppressionCalculatorOptions.ext] {
+      min_suppression_threshold: 0.3
+      overlap_type: INTERSECTION_OVER_UNION
+      algorithm: WEIGHTED
+    }
+  }
+}
+
+# Projects the detections from input tensor to the corresponding locations on
+# the original image (input to the graph).
+node {
+  calculator: "DetectionProjectionCalculator"
+  input_stream: "DETECTIONS:filtered_detections"
+  input_stream: "PROJECTION_MATRIX:transform_matrix"
+  output_stream: "DETECTIONS:detections"
+}

mediapipe/modules/face_detection/face_detection_front_by_roi_gpu.pbtxt

@@ -0,0 +1,143 @@
+# MediaPipe graph to detect faces. (CPU input, and inference is executed on
+# CPU.)
+#
+# It is required that "face_detection_front.tflite" is available at
+# "mediapipe/modules/face_detection/face_detection_front.tflite"
+# path during execution.
+#
+# EXAMPLE:
+#   node {
+#     calculator: "FaceDetectionFrontByRoiGpu"
+#     input_stream: "IMAGE:image"
+#     input_stream: "ROI:roi"
+#     output_stream: "DETECTIONS:face_detections"
+#   }
+
+type: "FaceDetectionFrontGpu"
+
+# GPU image. (GpuBuffer)
+input_stream: "IMAGE:image"
+
+# ROI (region of interest) within the given image where faces should be
+# detected. (NormalizedRect)
+input_stream: "ROI:roi"
+
+# Detected faces. (std::vector<Detection>)
+# NOTE: there will not be an output packet in the DETECTIONS stream for this
+# particular timestamp if none of faces detected. However, the MediaPipe
+# framework will internally inform the downstream calculators of the absence of
+# this packet so that they don't wait for it unnecessarily.
+output_stream: "DETECTIONS:detections"
+
+# Transforms specified region of image into 128x128 tensor keeping aspect ratio
+# (padding tensor if needed).
+node {
+  calculator: "ImageToTensorCalculator"
+  input_stream: "IMAGE_GPU:image"
+  input_stream: "NORM_RECT:roi"
+  output_stream: "TENSORS:input_tensors"
+  output_stream: "MATRIX:transform_matrix"
+  options: {
+    [mediapipe.ImageToTensorCalculatorOptions.ext] {
+      output_tensor_width: 128
+      output_tensor_height: 128
+      keep_aspect_ratio: true
+      output_tensor_float_range {
+        min: -1.0
+        max: 1.0
+      }
+      border_mode: BORDER_ZERO
+      gpu_origin: TOP_LEFT
+    }
+  }
+}
+
+# Runs a TensorFlow Lite model on GPU that takes an image tensor and outputs a
+# vector of tensors representing, for instance, detection boxes/keypoints and
+# scores.
+node {
+  calculator: "InferenceCalculator"
+  input_stream: "TENSORS:input_tensors"
+  output_stream: "TENSORS:detection_tensors"
+  options: {
+    [mediapipe.InferenceCalculatorOptions.ext] {
+      model_path: "mediapipe/modules/face_detection/face_detection_front.tflite"
+    }
+  }
+}
+
+# Generates a single side packet containing a vector of SSD anchors based on
+# the specification in the options.
+node {
+  calculator: "SsdAnchorsCalculator"
+  output_side_packet: "anchors"
+  options: {
+    [mediapipe.SsdAnchorsCalculatorOptions.ext] {
+      num_layers: 4
+      min_scale: 0.1484375
+      max_scale: 0.75
+      input_size_height: 128
+      input_size_width: 128
+      anchor_offset_x: 0.5
+      anchor_offset_y: 0.5
+      strides: 8
+      strides: 16
+      strides: 16
+      strides: 16
+      aspect_ratios: 1.0
+      fixed_anchor_size: true
+    }
+  }
+}
+
+# Decodes the detection tensors generated by the TensorFlow Lite model, based on
+# the SSD anchors and the specification in the options, into a vector of
+# detections. Each detection describes a detected object.
+node {
+  calculator: "TensorsToDetectionsCalculator"
+  input_stream: "TENSORS:detection_tensors"
+  input_side_packet: "ANCHORS:anchors"
+  output_stream: "DETECTIONS:unfiltered_detections"
+  options: {
+    [mediapipe.TensorsToDetectionsCalculatorOptions.ext] {
+      num_classes: 1
+      num_boxes: 896
+      num_coords: 16
+      box_coord_offset: 0
+      keypoint_coord_offset: 4
+      num_keypoints: 6
+      num_values_per_keypoint: 2
+      sigmoid_score: true
+      score_clipping_thresh: 100.0
+      reverse_output_order: true
+      x_scale: 128.0
+      y_scale: 128.0
+      h_scale: 128.0
+      w_scale: 128.0
+      min_score_thresh: 0.5
+    }
+  }
+}
+
+# Performs non-max suppression to remove excessive detections.
+node {
+  calculator: "NonMaxSuppressionCalculator"
+  input_stream: "unfiltered_detections"
+  output_stream: "filtered_detections"
+  options: {
+    [mediapipe.NonMaxSuppressionCalculatorOptions.ext] {
+      min_suppression_threshold: 0.3
+      overlap_type: INTERSECTION_OVER_UNION
+      algorithm: WEIGHTED
+    }
+  }
+}
+
+# Projects the detections from input tensor to the corresponding locations on
+# the original image (input to the graph).
+node {
+  calculator: "DetectionProjectionCalculator"
+  input_stream: "DETECTIONS:filtered_detections"
+  input_stream: "PROJECTION_MATRIX:transform_matrix"
+  output_stream: "DETECTIONS:detections"
+}

mediapipe/modules/face_detection/face_detection_front_cpu.pbtxt

@@ -41,6 +41,7 @@ node: {
         min: -1.0
         max: 1.0
       }
+      border_mode: BORDER_ZERO
     }
   }
 }

mediapipe/modules/face_detection/face_detection_front_gpu.pbtxt

@@ -41,6 +41,7 @@ node: {
         min: -1.0
         max: 1.0
       }
+      border_mode: BORDER_ZERO
       gpu_origin: TOP_LEFT
     }
   }

mediapipe/modules/face_landmark/BUILD

@@ -63,14 +63,13 @@ mediapipe_simple_subgraph(
         ":face_landmark_landmarks_to_roi",
         "//mediapipe/calculators/core:begin_loop_calculator",
         "//mediapipe/calculators/core:clip_vector_size_calculator",
+        "//mediapipe/calculators/core:constant_side_packet_calculator",
         "//mediapipe/calculators/core:end_loop_calculator",
         "//mediapipe/calculators/core:gate_calculator",
-        "//mediapipe/calculators/core:merge_calculator",
         "//mediapipe/calculators/core:previous_loopback_calculator",
         "//mediapipe/calculators/image:image_properties_calculator",
         "//mediapipe/calculators/util:association_norm_rect_calculator",
         "//mediapipe/calculators/util:collection_has_min_size_calculator",
-        "//mediapipe/calculators/util:logic_calculator",
         "//mediapipe/modules/face_detection:face_detection_front_cpu",
     ],
 )
@@ -85,14 +84,13 @@ mediapipe_simple_subgraph(
         ":face_landmark_landmarks_to_roi",
         "//mediapipe/calculators/core:begin_loop_calculator",
         "//mediapipe/calculators/core:clip_vector_size_calculator",
+        "//mediapipe/calculators/core:constant_side_packet_calculator",
         "//mediapipe/calculators/core:end_loop_calculator",
         "//mediapipe/calculators/core:gate_calculator",
-        "//mediapipe/calculators/core:merge_calculator",
         "//mediapipe/calculators/core:previous_loopback_calculator",
         "//mediapipe/calculators/image:image_properties_calculator",
         "//mediapipe/calculators/util:association_norm_rect_calculator",
         "//mediapipe/calculators/util:collection_has_min_size_calculator",
-        "//mediapipe/calculators/util:logic_calculator",
         "//mediapipe/modules/face_detection:face_detection_front_gpu",
     ],
 )

mediapipe/modules/face_landmark/face_detection_front_detection_to_roi.pbtxt

@@ -26,7 +26,6 @@ node {
       rotation_vector_start_keypoint_index: 0  # Left eye.
       rotation_vector_end_keypoint_index: 1  # Right eye.
       rotation_vector_target_angle_degrees: 0
-      output_zero_rect_for_empty_detections: true
     }
   }
 }

mediapipe/modules/face_landmark/face_landmark_cpu.pbtxt

@@ -81,6 +81,11 @@ node {
   calculator: "TensorsToFloatsCalculator"
   input_stream: "TENSORS:face_flag_tensor"
   output_stream: "FLOAT:face_presence_score"
+  options {
+    [mediapipe.TensorsToFloatsCalculatorOptions.ext] {
+      activation: SIGMOID
+    }
+  }
 }
 
 # Applies a threshold to the confidence score to determine whether a face is
@@ -91,7 +96,7 @@ node {
   output_stream: "FLAG:face_presence"
   options: {
     [mediapipe.ThresholdingCalculatorOptions.ext] {
-      threshold: 0.1
+      threshold: 0.5
     }
   }
 }

mediapipe/modules/face_landmark/face_landmark_front_cpu.pbtxt

@@ -26,10 +26,6 @@ input_stream: "IMAGE:image"
 # Max number of faces to detect/track. (int)
 input_side_packet: "NUM_FACES:num_faces"
 
-# Whether face detection can be skipped when face regions can already be
-# approximated from face landmarks on the previous frame.
-input_side_packet: "CAN_SKIP_DETECTION:can_skip_detection"
-
 # Collection of detected/predicted faces, each represented as a list of 468 face
 # landmarks. (std::vector<NormalizedLandmarkList>)
 # NOTE: there will not be an output packet in the LANDMARKS stream for this
@@ -48,32 +44,41 @@ output_stream: "ROIS_FROM_LANDMARKS:face_rects_from_landmarks"
 # (std::vector<NormalizedRect>)
 output_stream: "ROIS_FROM_DETECTIONS:face_rects_from_detections"
 
+# Defines whether landmarks on the previous image should be used to help
+# localize landmarks on the current image.
+node {
+  name: "ConstantSidePacketCalculator"
+  calculator: "ConstantSidePacketCalculator"
+  output_side_packet: "PACKET:use_prev_landmarks"
+  options: {
+    [mediapipe.ConstantSidePacketCalculatorOptions.ext]: {
+      packet { bool_value: true }
+    }
+  }
+}
+node {
+  calculator: "GateCalculator"
+  input_side_packet: "ALLOW:use_prev_landmarks"
+  input_stream: "prev_face_rects_from_landmarks"
+  output_stream: "gated_prev_face_rects_from_landmarks"
+}
+
 # Determines if an input vector of NormalizedRect has a size greater than or
 # equal to the provided num_faces.
 node {
   calculator: "NormalizedRectVectorHasMinSizeCalculator"
-  input_stream: "ITERABLE:prev_face_rects_from_landmarks"
+  input_stream: "ITERABLE:gated_prev_face_rects_from_landmarks"
   input_side_packet: "num_faces"
   output_stream: "prev_has_enough_faces"
 }
 
-# Drops the incoming image if FaceLandmarkCpu was able to identify face presence
-# in the previous image and skipping face detection is enabled. Otherwise,
-# passes the incoming image through to trigger a new round of face detection
-# in FaceDetectionFrontCpu.
-node {
-  calculator: "LogicCalculator"
-  options: {
-    [mediapipe.LogicCalculatorOptions.ext] { op: AND }
-  }
-  input_side_packet: "can_skip_detection"
-  input_stream: "prev_has_enough_faces"
-  output_stream: "skip_detection"
-}
+# Drops the incoming image if enough faces have already been identified from the
+# previous image. Otherwise, passes the incoming image through to trigger a new
+# round of face detection.
 node {
   calculator: "GateCalculator"
   input_stream: "image"
-  input_stream: "DISALLOW:skip_detection"
+  input_stream: "DISALLOW:prev_has_enough_faces"
   output_stream: "gated_image"
   options: {
     [mediapipe.GateCalculatorOptions.ext] {
@@ -81,12 +86,6 @@ node {
     }
   }
 }
-node {
-  calculator: "GateCalculator"
-  input_stream: "prev_face_rects_from_landmarks"
-  input_stream: "ALLOW:skip_detection"
-  output_stream: "gated_prev_face_rects_from_landmarks"
-}
 
 # Detects faces.
 node {
@@ -149,8 +148,8 @@ node {
 # overlapping regions based on the specified min_similarity_threshold.
 node {
   calculator: "AssociationNormRectCalculator"
-  input_stream: "gated_prev_face_rects_from_landmarks"
   input_stream: "face_rects_from_detections"
+  input_stream: "gated_prev_face_rects_from_landmarks"
   output_stream: "face_rects"
   options: {
     [mediapipe.AssociationCalculatorOptions.ext] {

mediapipe/modules/face_landmark/face_landmark_front_gpu.pbtxt

@@ -26,10 +26,6 @@ input_stream: "IMAGE:image"
 # Max number of faces to detect/track. (int)
 input_side_packet: "NUM_FACES:num_faces"
 
-# Whether face detection can be skipped when face regions can already be
-# approximated from face landmarks on the previous frame.
-input_side_packet: "CAN_SKIP_DETECTION:can_skip_detection"
-
 # Collection of detected/predicted faces, each represented as a list of 468 face
 # landmarks. (std::vector<NormalizedLandmarkList>)
 # NOTE: there will not be an output packet in the LANDMARKS stream for this
@@ -48,32 +44,41 @@ output_stream: "ROIS_FROM_LANDMARKS:face_rects_from_landmarks"
 # (std::vector<NormalizedRect>)
 output_stream: "ROIS_FROM_DETECTIONS:face_rects_from_detections"
 
+# Defines whether landmarks on the previous image should be used to help
+# localize landmarks on the current image.
+node {
+  name: "ConstantSidePacketCalculator"
+  calculator: "ConstantSidePacketCalculator"
+  output_side_packet: "PACKET:use_prev_landmarks"
+  options: {
+    [mediapipe.ConstantSidePacketCalculatorOptions.ext]: {
+      packet { bool_value: true }
+    }
+  }
+}
+node {
+  calculator: "GateCalculator"
+  input_side_packet: "ALLOW:use_prev_landmarks"
+  input_stream: "prev_face_rects_from_landmarks"
+  output_stream: "gated_prev_face_rects_from_landmarks"
+}
+
 # Determines if an input vector of NormalizedRect has a size greater than or
 # equal to the provided num_faces.
 node {
   calculator: "NormalizedRectVectorHasMinSizeCalculator"
-  input_stream: "ITERABLE:prev_face_rects_from_landmarks"
+  input_stream: "ITERABLE:gated_prev_face_rects_from_landmarks"
   input_side_packet: "num_faces"
   output_stream: "prev_has_enough_faces"
 }
 
-# Drops the incoming image if FaceLandmarkGpu was able to identify face presence
-# in the previous image and skipping face detection is enabled. Otherwise,
-# passes the incoming image through to trigger a new round of face detection
-# in FaceDetectionFrontGpu.
-node {
-  calculator: "LogicCalculator"
-  options: {
-    [mediapipe.LogicCalculatorOptions.ext] { op: AND }
-  }
-  input_side_packet: "can_skip_detection"
-  input_stream: "prev_has_enough_faces"
-  output_stream: "skip_detection"
-}
+# Drops the incoming image if enough faces have already been identified from the
+# previous image. Otherwise, passes the incoming image through to trigger a new
+# round of face detection.
 node {
   calculator: "GateCalculator"
   input_stream: "image"
-  input_stream: "DISALLOW:skip_detection"
+  input_stream: "DISALLOW:prev_has_enough_faces"
   output_stream: "gated_image"
   options: {
     [mediapipe.GateCalculatorOptions.ext] {
@@ -81,12 +86,6 @@ node {
     }
   }
 }
-node {
-  calculator: "GateCalculator"
-  input_stream: "prev_face_rects_from_landmarks"
-  input_stream: "ALLOW:skip_detection"
-  output_stream: "gated_prev_face_rects_from_landmarks"
-}
 
 # Detects faces.
 node {
@@ -149,8 +148,8 @@ node {
 # overlapping regions based on the specified min_similarity_threshold.
 node {
   calculator: "AssociationNormRectCalculator"
-  input_stream: "gated_prev_face_rects_from_landmarks"
   input_stream: "face_rects_from_detections"
+  input_stream: "gated_prev_face_rects_from_landmarks"
   output_stream: "face_rects"
   options: {
     [mediapipe.AssociationCalculatorOptions.ext] {

mediapipe/modules/face_landmark/face_landmark_gpu.pbtxt

@@ -81,6 +81,11 @@ node {
   calculator: "TensorsToFloatsCalculator"
   input_stream: "TENSORS:face_flag_tensor"
   output_stream: "FLOAT:face_presence_score"
+    options {
+      [mediapipe.TensorsToFloatsCalculatorOptions.ext] {
+        activation: SIGMOID
+      }
+    }
 }
 
 # Applies a threshold to the confidence score to determine whether a face is
@@ -91,7 +96,7 @@ node {
   output_stream: "FLAG:face_presence"
   options: {
     [mediapipe.ThresholdingCalculatorOptions.ext] {
-      threshold: 0.1
+      threshold: 0.5
     }
   }
 }

mediapipe/modules/hand_landmark/BUILD

@@ -40,7 +40,6 @@ mediapipe_simple_subgraph(
         "//mediapipe/calculators/tensor:tensors_to_landmarks_calculator",
         "//mediapipe/calculators/util:landmark_letterbox_removal_calculator",
         "//mediapipe/calculators/util:landmark_projection_calculator",
-        "//mediapipe/calculators/util:logic_calculator",
         "//mediapipe/calculators/util:thresholding_calculator",
     ],
 )
@@ -59,7 +58,6 @@ mediapipe_simple_subgraph(
         "//mediapipe/calculators/tensor:tensors_to_landmarks_calculator",
         "//mediapipe/calculators/util:landmark_letterbox_removal_calculator",
         "//mediapipe/calculators/util:landmark_projection_calculator",
-        "//mediapipe/calculators/util:logic_calculator",
         "//mediapipe/calculators/util:thresholding_calculator",
     ],
 )
@@ -74,6 +72,7 @@ mediapipe_simple_subgraph(
         ":palm_detection_detection_to_roi",
         "//mediapipe/calculators/core:begin_loop_calculator",
         "//mediapipe/calculators/core:clip_vector_size_calculator",
+        "//mediapipe/calculators/core:constant_side_packet_calculator",
         "//mediapipe/calculators/core:end_loop_calculator",
         "//mediapipe/calculators/core:flow_limiter_calculator",
         "//mediapipe/calculators/core:gate_calculator",
@@ -96,6 +95,7 @@ mediapipe_simple_subgraph(
         ":palm_detection_detection_to_roi",
         "//mediapipe/calculators/core:begin_loop_calculator",
         "//mediapipe/calculators/core:clip_vector_size_calculator",
+        "//mediapipe/calculators/core:constant_side_packet_calculator",
         "//mediapipe/calculators/core:end_loop_calculator",
         "//mediapipe/calculators/core:flow_limiter_calculator",
         "//mediapipe/calculators/core:gate_calculator",

mediapipe/modules/hand_landmark/calculators/hand_landmarks_to_rect_calculator.cc

@@ -58,7 +58,7 @@ float ComputeRotation(const NormalizedLandmarkList& landmarks,
   return rotation;
 }
 
-::mediapipe::Status NormalizedLandmarkListToRect(
+mediapipe::Status NormalizedLandmarkListToRect(
     const NormalizedLandmarkList& landmarks,
     const std::pair<int, int>& image_size, NormalizedRect* rect) {
   const float rotation = ComputeRotation(landmarks, image_size);
@@ -117,7 +117,7 @@ float ComputeRotation(const NormalizedLandmarkList& landmarks,
   rect->set_height(height);
   rect->set_rotation(rotation);
 
-  return ::mediapipe::OkStatus();
+  return mediapipe::OkStatus();
 }
 
 }  // namespace
@@ -130,21 +130,21 @@ float ComputeRotation(const NormalizedLandmarkList& landmarks,
 // mean of PIP joints at the top.
 class HandLandmarksToRectCalculator : public CalculatorBase {
  public:
-  static ::mediapipe::Status GetContract(CalculatorContract* cc) {
+  static mediapipe::Status GetContract(CalculatorContract* cc) {
     cc->Inputs().Tag(kNormalizedLandmarksTag).Set<NormalizedLandmarkList>();
     cc->Inputs().Tag(kImageSizeTag).Set<std::pair<int, int>>();
     cc->Outputs().Tag(kNormRectTag).Set<NormalizedRect>();
-    return ::mediapipe::OkStatus();
+    return mediapipe::OkStatus();
   }
 
-  ::mediapipe::Status Open(CalculatorContext* cc) override {
+  mediapipe::Status Open(CalculatorContext* cc) override {
     cc->SetOffset(TimestampDiff(0));
-    return ::mediapipe::OkStatus();
+    return mediapipe::OkStatus();
   }
 
-  ::mediapipe::Status Process(CalculatorContext* cc) override {
+  mediapipe::Status Process(CalculatorContext* cc) override {
     if (cc->Inputs().Tag(kNormalizedLandmarksTag).IsEmpty()) {
-      return ::mediapipe::OkStatus();
+      return mediapipe::OkStatus();
     }
     RET_CHECK(!cc->Inputs().Tag(kImageSizeTag).IsEmpty());
 
@@ -159,7 +159,7 @@ class HandLandmarksToRectCalculator : public CalculatorBase {
         .Tag(kNormRectTag)
         .Add(output_rect.release(), cc->InputTimestamp());
 
-    return ::mediapipe::OkStatus();
+    return mediapipe::OkStatus();
   }
 };
 REGISTER_CALCULATOR(HandLandmarksToRectCalculator);

mediapipe/modules/hand_landmark/hand_landmark_tracking_cpu.pbtxt

@@ -14,10 +14,6 @@ input_stream: "IMAGE:image"
 # Max number of hands to detect/track. (int)
 input_side_packet: "NUM_HANDS:num_hands"
 
-# Whether palm detection can be skipped when hand regions can already be
-# approximated from hand landmarks on the previous frame.
-input_side_packet: "CAN_SKIP_DETECTION:can_skip_detection"
-
 # Collection of detected/predicted hands, each represented as a list of
 # landmarks. (std::vector<NormalizedLandmarkList>)
 # NOTE: there will not be an output packet in the LANDMARKS stream for this
@@ -42,33 +38,41 @@ output_stream: "HAND_ROIS_FROM_LANDMARKS:hand_rects"
 # (std::vector<NormalizedRect>)
 output_stream: "HAND_ROIS_FROM_PALM_DETECTIONS:hand_rects_from_palm_detections"
 
+# Defines whether landmarks on the previous image should be used to help
+# localize landmarks on the current image.
+node {
+  name: "ConstantSidePacketCalculator"
+  calculator: "ConstantSidePacketCalculator"
+  output_side_packet: "PACKET:use_prev_landmarks"
+  options: {
+    [mediapipe.ConstantSidePacketCalculatorOptions.ext]: {
+      packet { bool_value: true }
+    }
+  }
+}
+node {
+  calculator: "GateCalculator"
+  input_side_packet: "ALLOW:use_prev_landmarks"
+  input_stream: "prev_hand_rects_from_landmarks"
+  output_stream: "gated_prev_hand_rects_from_landmarks"
+}
+
 # Determines if an input vector of NormalizedRect has a size greater than or
 # equal to the provided num_hands.
 node {
   calculator: "NormalizedRectVectorHasMinSizeCalculator"
-  input_stream: "ITERABLE:prev_hand_rects_from_landmarks"
+  input_stream: "ITERABLE:gated_prev_hand_rects_from_landmarks"
   input_side_packet: "num_hands"
   output_stream: "prev_has_enough_hands"
 }
 
-# Drops the incoming image if the previous image had at least N hands.
-# and skipping palm detection is enabled.
-# Otherwise, passes the incoming image through to trigger a new round of palm
-# detection.
-node {
-  calculator: "LogicCalculator"
-  options: {
-    [mediapipe.LogicCalculatorOptions.ext] { op: AND }
-  }
-  input_side_packet: "can_skip_detection"
-  input_stream: "prev_has_enough_hands"
-  output_stream: "skip_detection"
-}
-
+# Drops the incoming image if enough hands have already been identified from the
+# previous image. Otherwise, passes the incoming image through to trigger a new
+# round of palm detection.
 node {
   calculator: "GateCalculator"
   input_stream: "image"
-  input_stream: "DISALLOW:skip_detection"
+  input_stream: "DISALLOW:prev_has_enough_hands"
   output_stream: "palm_detection_image"
   options: {
     [mediapipe.GateCalculatorOptions.ext] {
@@ -76,12 +80,6 @@ node {
     }
   }
 }
-node {
-  calculator: "GateCalculator"
-  input_stream: "prev_hand_rects_from_landmarks"
-  input_stream: "ALLOW:skip_detection"
-  output_stream: "gated_prev_hand_rects_from_landmarks"
-}
 
 # Detects palms.
 node {
@@ -143,8 +141,8 @@ node {
 # overlapping regions based on the specified min_similarity_threshold.
 node {
   calculator: "AssociationNormRectCalculator"
-  input_stream: "gated_prev_hand_rects_from_landmarks"
   input_stream: "hand_rects_from_palm_detections"
+  input_stream: "gated_prev_hand_rects_from_landmarks"
   output_stream: "hand_rects"
   options: {
     [mediapipe.AssociationCalculatorOptions.ext] {

mediapipe/modules/hand_landmark/hand_landmark_tracking_gpu.pbtxt

@@ -14,10 +14,6 @@ input_stream: "IMAGE:image"
 # Max number of hands to detect/track. (int)
 input_side_packet: "NUM_HANDS:num_hands"
 
-# Whether palm detection can be skipped when hand regions can already be
-# approximated from hand landmarks on the previous frame.
-input_side_packet: "CAN_SKIP_DETECTION:can_skip_detection"
-
 # Collection of detected/predicted hands, each represented as a list of
 # landmarks. (std::vector<NormalizedLandmarkList>)
 # NOTE: there will not be an output packet in the LANDMARKS stream for this
@@ -42,32 +38,41 @@ output_stream: "HAND_ROIS_FROM_LANDMARKS:hand_rects"
 # (std::vector<NormalizedRect>)
 output_stream: "HAND_ROIS_FROM_PALM_DETECTIONS:hand_rects_from_palm_detections"
 
+# Defines whether landmarks on the previous image should be used to help
+# localize landmarks on the current image.
+node {
+  name: "ConstantSidePacketCalculator"
+  calculator: "ConstantSidePacketCalculator"
+  output_side_packet: "PACKET:use_prev_landmarks"
+  options: {
+    [mediapipe.ConstantSidePacketCalculatorOptions.ext]: {
+      packet { bool_value: true }
+    }
+  }
+}
+node {
+  calculator: "GateCalculator"
+  input_side_packet: "ALLOW:use_prev_landmarks"
+  input_stream: "prev_hand_rects_from_landmarks"
+  output_stream: "gated_prev_hand_rects_from_landmarks"
+}
+
 # Determines if an input vector of NormalizedRect has a size greater than or
 # equal to the provided num_hands.
 node {
   calculator: "NormalizedRectVectorHasMinSizeCalculator"
-  input_stream: "ITERABLE:prev_hand_rects_from_landmarks"
+  input_stream: "ITERABLE:gated_prev_hand_rects_from_landmarks"
   input_side_packet: "num_hands"
   output_stream: "prev_has_enough_hands"
 }
 
-# Drops the incoming image if the previous image had at least N hands.
-# and skipping palm detection is enabled.
-# Otherwise, passes the incoming image through to trigger a new round of palm
-# detection.
-node {
-  calculator: "LogicCalculator"
-  options: {
-    [mediapipe.LogicCalculatorOptions.ext] { op: AND }
-  }
-  input_side_packet: "can_skip_detection"
-  input_stream: "prev_has_enough_hands"
-  output_stream: "skip_detection"
-}
+# Drops the incoming image if enough hands have already been identified from the
+# previous image. Otherwise, passes the incoming image through to trigger a new
+# round of palm detection.
 node {
   calculator: "GateCalculator"
   input_stream: "image"
-  input_stream: "DISALLOW:skip_detection"
+  input_stream: "DISALLOW:prev_has_enough_hands"
   output_stream: "palm_detection_image"
   options: {
     [mediapipe.GateCalculatorOptions.ext] {
@@ -75,12 +80,6 @@ node {
     }
   }
 }
-node {
-  calculator: "GateCalculator"
-  input_stream: "prev_hand_rects_from_landmarks"
-  input_stream: "ALLOW:skip_detection"
-  output_stream: "gated_prev_hand_rects_from_landmarks"
-}
 
 # Detects palms.
 node {
@@ -143,8 +142,8 @@ node {
 # overlapping regions based on the specified min_similarity_threshold.
 node {
   calculator: "AssociationNormRectCalculator"
-  input_stream: "gated_prev_hand_rects_from_landmarks"
   input_stream: "hand_rects_from_palm_detections"
+  input_stream: "gated_prev_hand_rects_from_landmarks"
   output_stream: "hand_rects"
   options: {
     [mediapipe.AssociationCalculatorOptions.ext] {

mediapipe/modules/holistic_landmark/BUILD

@@ -0,0 +1,267 @@
+# Copyright 2020 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load("//mediapipe/framework/tool:mediapipe_graph.bzl", "mediapipe_simple_subgraph")
+
+# TODO: revert to private.
+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])
+
+exports_files([
+    "hand_recrop.tflite",
+])
+
+mediapipe_simple_subgraph(
+    name = "face_landmarks_from_pose_gpu",
+    graph = "face_landmarks_from_pose_gpu.pbtxt",
+    register_as = "FaceLandmarksFromPoseGpu",
+    deps = [
+        ":face_detection_front_detections_to_roi",
+        ":face_landmarks_from_pose_to_recrop_roi",
+        ":face_tracking",
+        "//mediapipe/calculators/core:split_normalized_landmark_list_calculator",
+        "//mediapipe/calculators/image:image_properties_calculator",
+        "//mediapipe/modules/face_detection:face_detection_front_by_roi_gpu",
+        "//mediapipe/modules/face_landmark:face_landmark_gpu",
+    ],
+)
+
+mediapipe_simple_subgraph(
+    name = "face_landmarks_from_pose_cpu",
+    graph = "face_landmarks_from_pose_cpu.pbtxt",
+    register_as = "FaceLandmarksFromPoseCpu",
+    deps = [
+        ":face_detection_front_detections_to_roi",
+        ":face_landmarks_from_pose_to_recrop_roi",
+        ":face_tracking",
+        "//mediapipe/calculators/core:split_normalized_landmark_list_calculator",
+        "//mediapipe/calculators/image:image_properties_calculator",
+        "//mediapipe/modules/face_detection:face_detection_front_by_roi_cpu",
+        "//mediapipe/modules/face_landmark:face_landmark_cpu",
+    ],
+)
+
+mediapipe_simple_subgraph(
+    name = "face_landmarks_to_roi",
+    graph = "face_landmarks_to_roi.pbtxt",
+    register_as = "FaceLandmarksToRoi",
+    deps = [
+        "//mediapipe/calculators/util:detections_to_rects_calculator",
+        "//mediapipe/calculators/util:landmarks_to_detection_calculator",
+        "//mediapipe/calculators/util:rect_transformation_calculator",
+    ],
+)
+
+mediapipe_simple_subgraph(
+    name = "face_detection_front_detections_to_roi",
+    graph = "face_detection_front_detections_to_roi.pbtxt",
+    register_as = "FaceDetectionFrontDetectionsToRoi",
+    deps = [
+        "//mediapipe/calculators/util:detections_to_rects_calculator",
+        "//mediapipe/calculators/util:rect_transformation_calculator",
+    ],
+)
+
+mediapipe_simple_subgraph(
+    name = "face_tracking",
+    graph = "face_tracking.pbtxt",
+    register_as = "FaceTracking",
+    deps = [
+        ":face_landmarks_to_roi",
+        "//mediapipe/calculators/core:previous_loopback_calculator",
+        "//mediapipe/modules/holistic_landmark/calculators:roi_tracking_calculator",
+    ],
+)
+
+mediapipe_simple_subgraph(
+    name = "face_landmarks_from_pose_to_recrop_roi",
+    graph = "face_landmarks_from_pose_to_recrop_roi.pbtxt",
+    register_as = "FaceLandmarksFromPoseToRecropRoi",
+    deps = [
+        "//mediapipe/calculators/util:detections_to_rects_calculator",
+        "//mediapipe/calculators/util:landmarks_to_detection_calculator",
+        "//mediapipe/calculators/util:rect_transformation_calculator",
+    ],
+)
+
+mediapipe_simple_subgraph(
+    name = "hand_landmarks_from_pose_gpu",
+    graph = "hand_landmarks_from_pose_gpu.pbtxt",
+    register_as = "HandLandmarksFromPoseGpu",
+    deps = [
+        ":hand_landmarks_from_pose_to_recrop_roi",
+        ":hand_recrop_by_roi_gpu",
+        ":hand_tracking",
+        ":hand_visibility_from_hand_landmarks_from_pose",
+        "//mediapipe/calculators/core:gate_calculator",
+        "//mediapipe/calculators/image:image_properties_calculator",
+        "//mediapipe/modules/hand_landmark:hand_landmark_gpu",
+    ],
+)
+
+mediapipe_simple_subgraph(
+    name = "hand_landmarks_from_pose_cpu",
+    graph = "hand_landmarks_from_pose_cpu.pbtxt",
+    register_as = "HandLandmarksFromPoseCpu",
+    deps = [
+        ":hand_landmarks_from_pose_to_recrop_roi",
+        ":hand_recrop_by_roi_cpu",
+        ":hand_tracking",
+        ":hand_visibility_from_hand_landmarks_from_pose",
+        "//mediapipe/calculators/core:gate_calculator",
+        "//mediapipe/calculators/image:image_properties_calculator",
+        "//mediapipe/modules/hand_landmark:hand_landmark_cpu",
+    ],
+)
+
+mediapipe_simple_subgraph(
+    name = "hand_landmarks_to_roi",
+    graph = "hand_landmarks_to_roi.pbtxt",
+    register_as = "HandLandmarksToRoi",
+    deps = [
+        "//mediapipe/calculators/core:split_normalized_landmark_list_calculator",
+        "//mediapipe/calculators/util:rect_transformation_calculator",
+        "//mediapipe/modules/hand_landmark/calculators:hand_landmarks_to_rect_calculator",
+    ],
+)
+
+mediapipe_simple_subgraph(
+    name = "hand_recrop_by_roi_gpu",
+    graph = "hand_recrop_by_roi_gpu.pbtxt",
+    register_as = "HandRecropByRoiGpu",
+    deps = [
+        "//mediapipe/calculators/image:image_properties_calculator",
+        "//mediapipe/calculators/tensor:image_to_tensor_calculator",
+        "//mediapipe/calculators/tensor:inference_calculator",
+        "//mediapipe/calculators/tensor:tensors_to_landmarks_calculator",
+        "//mediapipe/calculators/util:alignment_points_to_rects_calculator",
+        "//mediapipe/calculators/util:landmark_letterbox_removal_calculator",
+        "//mediapipe/calculators/util:landmark_projection_calculator",
+        "//mediapipe/calculators/util:landmarks_to_detection_calculator",
+        "//mediapipe/calculators/util:rect_transformation_calculator",
+    ],
+)
+
+mediapipe_simple_subgraph(
+    name = "hand_recrop_by_roi_cpu",
+    graph = "hand_recrop_by_roi_cpu.pbtxt",
+    register_as = "HandRecropByRoiCpu",
+    deps = [
+        "//mediapipe/calculators/image:image_properties_calculator",
+        "//mediapipe/calculators/tensor:image_to_tensor_calculator",
+        "//mediapipe/calculators/tensor:inference_calculator",
+        "//mediapipe/calculators/tensor:tensors_to_landmarks_calculator",
+        "//mediapipe/calculators/util:alignment_points_to_rects_calculator",
+        "//mediapipe/calculators/util:landmark_letterbox_removal_calculator",
+        "//mediapipe/calculators/util:landmark_projection_calculator",
+        "//mediapipe/calculators/util:landmarks_to_detection_calculator",
+        "//mediapipe/calculators/util:rect_transformation_calculator",
+    ],
+)
+
+mediapipe_simple_subgraph(
+    name = "hand_tracking",
+    graph = "hand_tracking.pbtxt",
+    register_as = "HandTracking",
+    deps = [
+        ":hand_landmarks_to_roi",
+        "//mediapipe/calculators/core:previous_loopback_calculator",
+        "//mediapipe/modules/holistic_landmark/calculators:roi_tracking_calculator",
+    ],
+)
+
+# TODO: parametrize holistic_landmark graph with visibility and make private.
+mediapipe_simple_subgraph(
+    name = "hand_wrist_for_pose",
+    graph = "hand_wrist_for_pose.pbtxt",
+    register_as = "HandWristForPose",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/calculators/core:constant_side_packet_calculator",
+        "//mediapipe/calculators/core:side_packet_to_stream_calculator",
+        "//mediapipe/calculators/core:split_normalized_landmark_list_calculator",
+        "//mediapipe/calculators/util:set_landmark_visibility_calculator",
+    ],
+)
+
+mediapipe_simple_subgraph(
+    name = "hand_landmarks_left_and_right_gpu",
+    graph = "hand_landmarks_left_and_right_gpu.pbtxt",
+    register_as = "HandLandmarksLeftAndRightGpu",
+    deps = [
+        ":hand_landmarks_from_pose_gpu",
+        "//mediapipe/calculators/core:split_normalized_landmark_list_calculator",
+    ],
+)
+
+mediapipe_simple_subgraph(
+    name = "hand_landmarks_left_and_right_cpu",
+    graph = "hand_landmarks_left_and_right_cpu.pbtxt",
+    register_as = "HandLandmarksLeftAndRightCpu",
+    deps = [
+        ":hand_landmarks_from_pose_cpu",
+        "//mediapipe/calculators/core:split_normalized_landmark_list_calculator",
+    ],
+)
+
+mediapipe_simple_subgraph(
+    name = "hand_landmarks_from_pose_to_recrop_roi",
+    graph = "hand_landmarks_from_pose_to_recrop_roi.pbtxt",
+    register_as = "HandLandmarksFromPoseToRecropRoi",
+    deps = [
+        "//mediapipe/calculators/util:landmarks_to_detection_calculator",
+        "//mediapipe/calculators/util:rect_transformation_calculator",
+        "//mediapipe/modules/holistic_landmark/calculators:hand_detections_from_pose_to_rects_calculator",
+    ],
+)
+
+mediapipe_simple_subgraph(
+    name = "hand_visibility_from_hand_landmarks_from_pose",
+    graph = "hand_visibility_from_hand_landmarks_from_pose.pbtxt",
+    register_as = "HandVisibilityFromHandLandmarksFromPose",
+    deps = [
+        "//mediapipe/calculators/core:split_normalized_landmark_list_calculator",
+        "//mediapipe/calculators/util:landmark_visibility_calculator",
+        "//mediapipe/calculators/util:thresholding_calculator",
+    ],
+)
+
+mediapipe_simple_subgraph(
+    name = "holistic_landmark_gpu",
+    graph = "holistic_landmark_gpu.pbtxt",
+    register_as = "HolisticLandmarkGpu",
+    visibility = ["//visibility:public"],
+    deps = [
+        ":face_landmarks_from_pose_gpu",
+        ":hand_landmarks_left_and_right_gpu",
+        "//mediapipe/calculators/core:flow_limiter_calculator",
+        "//mediapipe/calculators/image:image_transformation_calculator",
+        "//mediapipe/modules/pose_landmark:pose_landmark_gpu",
+    ],
+)
+
+mediapipe_simple_subgraph(
+    name = "holistic_landmark_cpu",
+    graph = "holistic_landmark_cpu.pbtxt",
+    register_as = "HolisticLandmarkCpu",
+    visibility = ["//visibility:public"],
+    deps = [
+        ":face_landmarks_from_pose_cpu",
+        ":hand_landmarks_left_and_right_cpu",
+        "//mediapipe/calculators/core:flow_limiter_calculator",
+        "//mediapipe/calculators/image:image_transformation_calculator",
+        "//mediapipe/modules/pose_landmark:pose_landmark_cpu",
+    ],
+)

mediapipe/modules/holistic_landmark/README.md

@@ -0,0 +1,6 @@
+# holistic_landmark
+
+Subgraphs|Details
+:--- | :---
+[`HolisticLandmarkCpu`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/holistic_landmark/holistic_landmark_cpu.pbtxt)| Predicts pose + left/right hand + face landmarks. (CPU input)
+[`HolisticLandmarkCpu`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/holistic_landmark/holistic_landmark_gpu.pbtxt)| Predicts pose + left/right hand + face landmarks. (GPU input.)

mediapipe/modules/holistic_landmark/calculators/BUILD

@@ -0,0 +1,63 @@
+# Copyright 2020 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library")
+
+licenses(["notice"])
+
+package(default_visibility = ["//visibility:public"])
+
+cc_library(
+    name = "hand_detections_from_pose_to_rects_calculator",
+    srcs = ["hand_detections_from_pose_to_rects_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/calculators/util:detections_to_rects_calculator",
+        "//mediapipe/calculators/util:detections_to_rects_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:calculator_options_cc_proto",
+        "//mediapipe/framework/formats:detection_cc_proto",
+        "//mediapipe/framework/formats:location_data_cc_proto",
+        "//mediapipe/framework/formats:rect_cc_proto",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+    ],
+    alwayslink = 1,
+)
+
+mediapipe_proto_library(
+    name = "roi_tracking_calculator_proto",
+    srcs = ["roi_tracking_calculator.proto"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_options_proto",
+        "//mediapipe/framework:calculator_proto",
+    ],
+)
+
+cc_library(
+    name = "roi_tracking_calculator",
+    srcs = ["roi_tracking_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":roi_tracking_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:landmark_cc_proto",
+        "//mediapipe/framework/formats:rect_cc_proto",
+        "//mediapipe/framework/port:logging",
+        "//mediapipe/framework/port:rectangle",
+        "@com_google_absl//absl/strings:str_format",
+    ],
+    alwayslink = 1,
+)

mediapipe/modules/holistic_landmark/calculators/hand_detections_from_pose_to_rects_calculator.cc

@@ -0,0 +1,156 @@
+#include <cmath>
+
+#include "mediapipe/calculators/util/detections_to_rects_calculator.h"
+#include "mediapipe/calculators/util/detections_to_rects_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/calculator_options.pb.h"
+#include "mediapipe/framework/formats/detection.pb.h"
+#include "mediapipe/framework/formats/location_data.pb.h"
+#include "mediapipe/framework/formats/rect.pb.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+
+namespace {}  // namespace
+
+// Generates a hand ROI based on a hand detection derived from hand-related pose
+// landmarks.
+//
+// Inputs:
+//   DETECTION - Detection.
+//     Detection to convert to ROI. Must contain 3 key points indicating: wrist,
+//     pinky and index fingers.
+//
+//   IMAGE_SIZE - std::pair<int, int>
+//     Image width and height.
+//
+// Outputs:
+//   NORM_RECT - NormalizedRect.
+//     ROI based on passed input.
+//
+// Examples
+// node {
+//   calculator: "HandDetectionsFromPoseToRectsCalculator"
+//   input_stream: "DETECTION:hand_detection_from_pose"
+//   input_stream: "IMAGE_SIZE:image_size"
+//   output_stream: "NORM_RECT:hand_roi_from_pose"
+// }
+class HandDetectionsFromPoseToRectsCalculator
+    : public DetectionsToRectsCalculator {
+ public:
+  ::mediapipe::Status Open(CalculatorContext* cc) override;
+
+ private:
+  ::mediapipe::Status DetectionToNormalizedRect(
+      const Detection& detection, const DetectionSpec& detection_spec,
+      NormalizedRect* rect) override;
+  ::mediapipe::Status ComputeRotation(const Detection& detection,
+                                      const DetectionSpec& detection_spec,
+                                      float* rotation) override;
+};
+REGISTER_CALCULATOR(HandDetectionsFromPoseToRectsCalculator);
+
+namespace {
+
+constexpr int kWrist = 0;
+constexpr int kPinky = 1;
+constexpr int kIndex = 2;
+
+constexpr char kImageSizeTag[] = "IMAGE_SIZE";
+
+}  // namespace
+
+::mediapipe::Status HandDetectionsFromPoseToRectsCalculator::Open(
+    CalculatorContext* cc) {
+  RET_CHECK(cc->Inputs().HasTag(kImageSizeTag))
+      << "Image size is required to calculate rotated rect.";
+  cc->SetOffset(TimestampDiff(0));
+  target_angle_ = M_PI * 0.5f;
+  rotate_ = true;
+  options_ = cc->Options<DetectionsToRectsCalculatorOptions>();
+  output_zero_rect_for_empty_detections_ =
+      options_.output_zero_rect_for_empty_detections();
+
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status
+HandDetectionsFromPoseToRectsCalculator ::DetectionToNormalizedRect(
+    const Detection& detection, const DetectionSpec& detection_spec,
+    NormalizedRect* rect) {
+  const auto& location_data = detection.location_data();
+  const auto& image_size = detection_spec.image_size;
+  RET_CHECK(image_size) << "Image size is required to calculate rotation";
+
+  const float x_wrist =
+      location_data.relative_keypoints(kWrist).x() * image_size->first;
+  const float y_wrist =
+      location_data.relative_keypoints(kWrist).y() * image_size->second;
+
+  const float x_index =
+      location_data.relative_keypoints(kIndex).x() * image_size->first;
+  const float y_index =
+      location_data.relative_keypoints(kIndex).y() * image_size->second;
+
+  const float x_pinky =
+      location_data.relative_keypoints(kPinky).x() * image_size->first;
+  const float y_pinky =
+      location_data.relative_keypoints(kPinky).y() * image_size->second;
+
+  // Estimate middle finger.
+  const float x_middle = (2.f * x_index + x_pinky) / 3.f;
+  const float y_middle = (2.f * y_index + y_pinky) / 3.f;
+
+  // Crop center as middle finger.
+  const float center_x = x_middle;
+  const float center_y = y_middle;
+
+  // Bounding box size as double distance from middle finger to wrist.
+  const float box_size =
+      std::sqrt((x_middle - x_wrist) * (x_middle - x_wrist) +
+                (y_middle - y_wrist) * (y_middle - y_wrist)) *
+      2.0;
+
+  // Set resulting bounding box.
+  rect->set_x_center(center_x / image_size->first);
+  rect->set_y_center(center_y / image_size->second);
+  rect->set_width(box_size / image_size->first);
+  rect->set_height(box_size / image_size->second);
+
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status HandDetectionsFromPoseToRectsCalculator::ComputeRotation(
+    const Detection& detection, const DetectionSpec& detection_spec,
+    float* rotation) {
+  const auto& location_data = detection.location_data();
+  const auto& image_size = detection_spec.image_size;
+  RET_CHECK(image_size) << "Image size is required to calculate rotation";
+
+  const float x_wrist =
+      location_data.relative_keypoints(kWrist).x() * image_size->first;
+  const float y_wrist =
+      location_data.relative_keypoints(kWrist).y() * image_size->second;
+
+  const float x_index =
+      location_data.relative_keypoints(kIndex).x() * image_size->first;
+  const float y_index =
+      location_data.relative_keypoints(kIndex).y() * image_size->second;
+
+  const float x_pinky =
+      location_data.relative_keypoints(kPinky).x() * image_size->first;
+  const float y_pinky =
+      location_data.relative_keypoints(kPinky).y() * image_size->second;
+
+  // Estimate middle finger.
+  const float x_middle = (2.f * x_index + x_pinky) / 3.f;
+  const float y_middle = (2.f * y_index + y_pinky) / 3.f;
+
+  *rotation = NormalizeRadians(
+      target_angle_ - std::atan2(-(y_middle - y_wrist), x_middle - x_wrist));
+
+  return ::mediapipe::OkStatus();
+}
+
+}  // namespace mediapipe

mediapipe/modules/holistic_landmark/calculators/roi_tracking_calculator.cc

@@ -0,0 +1,358 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <math.h>
+
+#include <cstdlib>
+
+#include "absl/strings/str_format.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/landmark.pb.h"
+#include "mediapipe/framework/formats/rect.pb.h"
+#include "mediapipe/framework/port/logging.h"
+#include "mediapipe/framework/port/rectangle.h"
+#include "mediapipe/modules/holistic_landmark/calculators/roi_tracking_calculator.pb.h"
+
+namespace mediapipe {
+
+namespace {
+
+constexpr char kPrevLandmarksTag[] = "PREV_LANDMARKS";
+constexpr char kPrevLandmarksRectTag[] = "PREV_LANDMARKS_RECT";
+constexpr char kRecropRectTag[] = "RECROP_RECT";
+constexpr char kImageSizeTag[] = "IMAGE_SIZE";
+constexpr char kTrackingRectTag[] = "TRACKING_RECT";
+
+// TODO: Use rect rotation.
+// Verifies that Intersection over Union of previous frame rect and current
+// frame re-crop rect is less than threshold.
+bool IouRequirementsSatisfied(const NormalizedRect& prev_rect,
+                              const NormalizedRect& recrop_rect,
+                              const std::pair<int, int>& image_size,
+                              const float min_iou) {
+  auto r1 = Rectangle_f(prev_rect.x_center() * image_size.first,
+                        prev_rect.y_center() * image_size.second,
+                        prev_rect.width() * image_size.first,
+                        prev_rect.height() * image_size.second);
+  auto r2 = Rectangle_f(recrop_rect.x_center() * image_size.first,
+                        recrop_rect.y_center() * image_size.second,
+                        recrop_rect.width() * image_size.first,
+                        recrop_rect.height() * image_size.second);
+
+  const float intersection_area = r1.Intersect(r2).Area();
+  const float union_area = r1.Area() + r2.Area() - intersection_area;
+
+  const float intersection_threshold = union_area * min_iou;
+  if (intersection_area < intersection_threshold) {
+    VLOG(1) << absl::StrFormat("Lost tracking: IoU intersection %f < %f",
+                               intersection_area, intersection_threshold);
+    return false;
+  }
+
+  return true;
+}
+
+// Verifies that current frame re-crop rect rotation/translation/scale didn't
+// change much comparing to the previous frame rect. Translation and scale are
+// normalized by current frame re-crop rect.
+bool RectRequirementsSatisfied(const NormalizedRect& prev_rect,
+                               const NormalizedRect& recrop_rect,
+                               const std::pair<int, int> image_size,
+                               const float rotation_degrees,
+                               const float translation, const float scale) {
+  // Rotate both rects so that re-crop rect edges are parallel to XY axes. That
+  // will allow to compute x/y translation of the previous frame rect along axes
+  // of the current frame re-crop rect.
+  const float rotation = -recrop_rect.rotation();
+
+  const float cosa = cos(rotation);
+  const float sina = sin(rotation);
+
+  // Rotate previous frame rect and get its parameters.
+  const float prev_rect_x = prev_rect.x_center() * image_size.first * cosa -
+                            prev_rect.y_center() * image_size.second * sina;
+  const float prev_rect_y = prev_rect.x_center() * image_size.first * sina +
+                            prev_rect.y_center() * image_size.second * cosa;
+  const float prev_rect_width = prev_rect.width() * image_size.first;
+  const float prev_rect_height = prev_rect.height() * image_size.second;
+  const float prev_rect_rotation = prev_rect.rotation() / M_PI * 180.f;
+
+  // Rotate current frame re-crop rect and get its parameters.
+  const float recrop_rect_x = recrop_rect.x_center() * image_size.first * cosa -
+                              recrop_rect.y_center() * image_size.second * sina;
+  const float recrop_rect_y = recrop_rect.x_center() * image_size.first * sina +
+                              recrop_rect.y_center() * image_size.second * cosa;
+  const float recrop_rect_width = recrop_rect.width() * image_size.first;
+  const float recrop_rect_height = recrop_rect.height() * image_size.second;
+  const float recrop_rect_rotation = recrop_rect.rotation() / M_PI * 180.f;
+
+  // Rect requirements are satisfied unless one of the checks below fails.
+  bool satisfied = true;
+
+  // Ensure that rotation diff is in [0, 180] range.
+  float rotation_diff = prev_rect_rotation - recrop_rect_rotation;
+  if (rotation_diff > 180.f) {
+    rotation_diff -= 360.f;
+  }
+  if (rotation_diff < -180.f) {
+    rotation_diff += 360.f;
+  }
+  rotation_diff = abs(rotation_diff);
+  if (rotation_diff > rotation_degrees) {
+    satisfied = false;
+    VLOG(1) << absl::StrFormat("Lost tracking: rect rotation %f > %f",
+                               rotation_diff, rotation_degrees);
+  }
+
+  const float x_diff = abs(prev_rect_x - recrop_rect_x);
+  const float x_threshold = recrop_rect_width * translation;
+  if (x_diff > x_threshold) {
+    satisfied = false;
+    VLOG(1) << absl::StrFormat("Lost tracking: rect x translation %f > %f",
+                               x_diff, x_threshold);
+  }
+
+  const float y_diff = abs(prev_rect_y - recrop_rect_y);
+  const float y_threshold = recrop_rect_height * translation;
+  if (y_diff > y_threshold) {
+    satisfied = false;
+    VLOG(1) << absl::StrFormat("Lost tracking: rect y translation %f > %f",
+                               y_diff, y_threshold);
+  }
+
+  const float width_diff = abs(prev_rect_width - recrop_rect_width);
+  const float width_threshold = recrop_rect_width * scale;
+  if (width_diff > width_threshold) {
+    satisfied = false;
+    VLOG(1) << absl::StrFormat("Lost tracking: rect width %f > %f", width_diff,
+                               width_threshold);
+  }
+
+  const float height_diff = abs(prev_rect_height - recrop_rect_height);
+  const float height_threshold = recrop_rect_height * scale;
+  if (height_diff > height_threshold) {
+    satisfied = false;
+    VLOG(1) << absl::StrFormat("Lost tracking: rect height %f > %f",
+                               height_diff, height_threshold);
+  }
+
+  return satisfied;
+}
+
+// Verifies that landmarks from the previous frame are within re-crop rectangle
+// bounds on the current frame.
+bool LandmarksRequirementsSatisfied(const NormalizedLandmarkList& landmarks,
+                                    const NormalizedRect& recrop_rect,
+                                    const std::pair<int, int> image_size,
+                                    const float recrop_rect_margin) {
+  // Rotate both re-crop rectangle and landmarks so that re-crop rectangle edges
+  // are parallel to XY axes. It will allow to easily check if landmarks are
+  // within re-crop rect bounds along re-crop rect axes.
+  //
+  // Rect rotation is specified clockwise. To apply cos/sin functions we
+  // transform it into counterclockwise.
+  const float rotation = -recrop_rect.rotation();
+
+  const float cosa = cos(rotation);
+  const float sina = sin(rotation);
+
+  // Rotate rect.
+  const float rect_x = recrop_rect.x_center() * image_size.first * cosa -
+                       recrop_rect.y_center() * image_size.second * sina;
+  const float rect_y = recrop_rect.x_center() * image_size.first * sina +
+                       recrop_rect.y_center() * image_size.second * cosa;
+  const float rect_width =
+      recrop_rect.width() * image_size.first * (1.f + recrop_rect_margin);
+  const float rect_height =
+      recrop_rect.height() * image_size.second * (1.f + recrop_rect_margin);
+
+  // Get rect bounds.
+  const float rect_left = rect_x - rect_width * 0.5f;
+  const float rect_right = rect_x + rect_width * 0.5f;
+  const float rect_top = rect_y - rect_height * 0.5f;
+  const float rect_bottom = rect_y + rect_height * 0.5f;
+
+  for (int i = 0; i < landmarks.landmark_size(); ++i) {
+    const auto& landmark = landmarks.landmark(i);
+    const float x = landmark.x() * image_size.first * cosa -
+                    landmark.y() * image_size.second * sina;
+    const float y = landmark.x() * image_size.first * sina +
+                    landmark.y() * image_size.second * cosa;
+
+    if (!(rect_left < x && x < rect_right && rect_top < y && y < rect_bottom)) {
+      VLOG(1) << "Lost tracking: landmarks out of re-crop rect";
+      return false;
+    }
+  }
+
+  return true;
+}
+
+}  // namespace
+
+// A calculator to track object rectangle between frames.
+//
+// Calculator checks that all requirements for tracking are satisfied and uses
+// rectangle from the previous frame in this case, otherwise - uses current
+// frame re-crop rectangle.
+//
+// There are several types of tracking requirements that can be configured via
+// options:
+//   IoU: Verifies that IoU of the previous frame rectangle and current frame
+//     re-crop rectangle is less than a given threshold.
+//   Rect parameters: Verifies that rotation/translation/scale of the re-crop
+//     rectangle on the current frame is close to the rectangle from the
+//     previous frame within given thresholds.
+//   Landmarks: Verifies that landmarks from the previous frame are within
+//     the re-crop rectangle on the current frame.
+//
+// Inputs:
+//   PREV_LANDMARKS: Object landmarks from the previous frame.
+//   PREV_LANDMARKS_RECT: Object rectangle based on the landmarks from the
+//     previous frame.
+//   RECROP_RECT: Object re-crop rectangle from the current frame.
+//   IMAGE_SIZE: Image size to transform normalized coordinates to absolute.
+//
+// Outputs:
+//   TRACKING_RECT: Rectangle to use for object prediction on the current frame.
+//     It will be either object rectangle from the previous frame (if all
+//     tracking requirements are satisfied) or re-crop rectangle from the
+//     current frame (if tracking lost the object).
+//
+// Example config:
+//   node {
+//     calculator: "RoiTrackingCalculator"
+//     input_stream: "PREV_LANDMARKS:prev_hand_landmarks"
+//     input_stream: "PREV_LANDMARKS_RECT:prev_hand_landmarks_rect"
+//     input_stream: "RECROP_RECT:hand_recrop_rect"
+//     input_stream: "IMAGE_SIZE:image_size"
+//     output_stream: "TRACKING_RECT:hand_tracking_rect"
+//     options: {
+//       [mediapipe.RoiTrackingCalculatorOptions.ext] {
+//         rect_requirements: {
+//           rotation_degrees: 40.0
+//           translation: 0.2
+//           scale: 0.4
+//         }
+//         landmarks_requirements: {
+//           recrop_rect_margin: -0.1
+//         }
+//       }
+//     }
+//   }
+class RoiTrackingCalculator : public CalculatorBase {
+ public:
+  static mediapipe::Status GetContract(CalculatorContract* cc);
+  mediapipe::Status Open(CalculatorContext* cc) override;
+  mediapipe::Status Process(CalculatorContext* cc) override;
+
+ private:
+  RoiTrackingCalculatorOptions options_;
+};
+REGISTER_CALCULATOR(RoiTrackingCalculator);
+
+mediapipe::Status RoiTrackingCalculator::GetContract(CalculatorContract* cc) {
+  cc->Inputs().Tag(kPrevLandmarksTag).Set<NormalizedLandmarkList>();
+  cc->Inputs().Tag(kPrevLandmarksRectTag).Set<NormalizedRect>();
+  cc->Inputs().Tag(kRecropRectTag).Set<NormalizedRect>();
+  cc->Inputs().Tag(kImageSizeTag).Set<std::pair<int, int>>();
+  cc->Outputs().Tag(kTrackingRectTag).Set<NormalizedRect>();
+
+  return mediapipe::OkStatus();
+}
+
+mediapipe::Status RoiTrackingCalculator::Open(CalculatorContext* cc) {
+  cc->SetOffset(TimestampDiff(0));
+  options_ = cc->Options<RoiTrackingCalculatorOptions>();
+  return mediapipe::OkStatus();
+}
+
+mediapipe::Status RoiTrackingCalculator::Process(CalculatorContext* cc) {
+  // If there is no current frame re-crop rect (i.e. object is not present on
+  // the current frame) - return empty packet.
+  if (cc->Inputs().Tag(kRecropRectTag).IsEmpty()) {
+    return mediapipe::OkStatus();
+  }
+
+  // If there is no previous rect, but there is current re-crop rect - return
+  // current re-crop rect as is.
+  if (cc->Inputs().Tag(kPrevLandmarksRectTag).IsEmpty()) {
+    cc->Outputs()
+        .Tag(kTrackingRectTag)
+        .AddPacket(cc->Inputs().Tag(kRecropRectTag).Value());
+    return mediapipe::OkStatus();
+  }
+
+  // At this point we have both previous rect (which also means we have previous
+  // landmarks) and currrent re-crop rect.
+  const auto& prev_landmarks =
+      cc->Inputs().Tag(kPrevLandmarksTag).Get<NormalizedLandmarkList>();
+  const auto& prev_rect =
+      cc->Inputs().Tag(kPrevLandmarksRectTag).Get<NormalizedRect>();
+  const auto& recrop_rect =
+      cc->Inputs().Tag(kRecropRectTag).Get<NormalizedRect>();
+  const auto& image_size =
+      cc->Inputs().Tag(kImageSizeTag).Get<std::pair<int, int>>();
+
+  // Keep tracking unless one of the requirements below is not satisfied.
+  bool keep_tracking = true;
+
+  // If IoU of the previous rect and current re-crop rect is lower than allowed
+  // threshold - use current re-crop rect.
+  if (options_.has_iou_requirements() &&
+      !IouRequirementsSatisfied(prev_rect, recrop_rect, image_size,
+                                options_.iou_requirements().min_iou())) {
+    keep_tracking = false;
+  }
+
+  // If previous rect and current re-crop rect differ more than it is allowed by
+  // the augmentations (used during the model training) - use current re-crop
+  // rect.
+  if (options_.has_rect_requirements() &&
+      !RectRequirementsSatisfied(
+          prev_rect, recrop_rect, image_size,
+          options_.rect_requirements().rotation_degrees(),
+          options_.rect_requirements().translation(),
+          options_.rect_requirements().scale())) {
+    keep_tracking = false;
+  }
+
+  // If landmarks from the previous frame are not in the current re-crop rect
+  // (i.e. object moved too fast and using previous frame rect won't cover
+  // landmarks on the current frame) - use current re-crop rect.
+  if (options_.has_landmarks_requirements() &&
+      !LandmarksRequirementsSatisfied(
+          prev_landmarks, recrop_rect, image_size,
+          options_.landmarks_requirements().recrop_rect_margin())) {
+    keep_tracking = false;
+  }
+
+  // If object didn't move a lot comparing to the previous frame - we'll keep
+  // tracking it and will return rect from the previous frame, otherwise -
+  // return re-crop rect from the current frame.
+  if (keep_tracking) {
+    cc->Outputs()
+        .Tag(kTrackingRectTag)
+        .AddPacket(cc->Inputs().Tag(kPrevLandmarksRectTag).Value());
+  } else {
+    cc->Outputs()
+        .Tag(kTrackingRectTag)
+        .AddPacket(cc->Inputs().Tag(kRecropRectTag).Value());
+    VLOG(1) << "Lost tracking: check messages above for details";
+  }
+
+  return mediapipe::OkStatus();
+}
+
+}  // namespace mediapipe

mediapipe/modules/holistic_landmark/calculators/roi_tracking_calculator.proto

@@ -0,0 +1,59 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+
+message RoiTrackingCalculatorOptions {
+  extend CalculatorOptions {
+    optional RoiTrackingCalculatorOptions ext = 329994630;
+  }
+
+  // Verifies that Intersection over Union of previous frame rect and current
+  // frame re-crop rect is less than threshold.
+  message IouRequirements {
+    optional float min_iou = 1 [default = 0.5];
+  }
+
+  // Verifies that current frame re-crop rect rotation/translation/scale didn't
+  // change much comparing to the previous frame rect.
+  message RectRequirements {
+    // Allowed rotation change defined in degrees.
+    optional float rotation_degrees = 1 [default = 10.0];
+
+    // Allowed translation change defined as absolute translation normalized by
+    // re-crop rectangle size.
+    optional float translation = 2 [default = 0.1];
+
+    // Allowed scale change defined as absolute translation normalized by
+    // re-crop rectangle size.
+    optional float scale = 3 [default = 0.1];
+  }
+
+  // Verifies that landmarks from the previous frame are within re-crop
+  // rectangle bounds on the current frame.
+  message LandmarksRequirements {
+    // Margin to apply to re-crop rectangle before checking verifing landmarks.
+    optional float recrop_rect_margin = 1 [default = 0.0];
+  }
+
+  optional IouRequirements iou_requirements = 1;
+
+  optional RectRequirements rect_requirements = 2;
+
+  optional LandmarksRequirements landmarks_requirements = 3;
+}

mediapipe/modules/holistic_landmark/face_detection_front_detections_to_roi.pbtxt

@@ -0,0 +1,48 @@
+# Calculates ROI from detections provided by `face_detection_front.tflite`
+# model.
+type: "FaceDetectionFrontDetectionsToRoi"
+
+# Detected faces. (std::vector<Detection>)
+input_stream: "DETECTIONS:detections"
+# Image size (width & height). (std::pair<int, int>)
+input_stream: "IMAGE_SIZE:image_size"
+
+# Refined (more accurate) ROI to use for face landmarks prediction.
+# (NormalizedRect)
+output_stream: "ROI:roi"
+
+# Converts the face detection into a rectangle (normalized by image size)
+# that encloses the face and is rotated such that the line connecting right side
+# of the right eye and left side of the left eye is aligned with the X-axis of
+# the rectangle.
+node {
+  calculator: "DetectionsToRectsCalculator"
+  input_stream: "DETECTIONS:detections"
+  input_stream: "IMAGE_SIZE:image_size"
+  output_stream: "NORM_RECT:raw_roi"
+  options: {
+    [mediapipe.DetectionsToRectsCalculatorOptions.ext] {
+      rotation_vector_start_keypoint_index: 0  # Right eye.
+      rotation_vector_end_keypoint_index: 1  # Left eye.
+      rotation_vector_target_angle_degrees: 0
+      conversion_mode: USE_KEYPOINTS
+    }
+  }
+}
+
+# Expands and shifts the rectangle that contains the face so that it's likely
+# to cover the entire face.
+node {
+  calculator: "RectTransformationCalculator"
+  input_stream: "NORM_RECT:raw_roi"
+  input_stream: "IMAGE_SIZE:image_size"
+  output_stream: "roi"
+  options: {
+    [mediapipe.RectTransformationCalculatorOptions.ext] {
+      scale_x: 2.0
+      scale_y: 2.0
+      shift_y: -0.1
+      square_long: true
+    }
+  }
+}

mediapipe/modules/holistic_landmark/face_landmarks_from_pose_cpu.pbtxt

@@ -0,0 +1,76 @@
+# Predicts face landmarks within an ROI derived from face-related pose
+# landmarks.
+
+type: "FaceLandmarksFromPoseCpu"
+
+# CPU image. (ImageFrame)
+input_stream: "IMAGE:input_video"
+# Face-related pose landmarks. (NormalizedLandmarkList)
+input_stream: "FACE_LANDMARKS_FROM_POSE:face_landmarks_from_pose"
+
+# Face landmarks. (NormalizedLandmarkList)
+output_stream: "FACE_LANDMARKS:face_landmarks"
+
+# Debug outputs.
+# Face ROI derived from face-related pose landmarks, which defines the search
+# region for the face detection model. (NormalizedRect)
+output_stream: "FACE_ROI_FROM_POSE:face_roi_from_pose"
+# Refined face crop rectangle predicted by face detection model.
+# (NormalizedRect)
+output_stream: "FACE_ROI_FROM_DETECTION:face_roi_from_detection"
+# Rectangle used to predict face landmarks. (NormalizedRect)
+output_stream: "FACE_TRACKING_ROI:face_tracking_roi"
+
+# TODO: do not predict face when most of the face landmarks from
+# pose are invisible.
+
+# Extracts image size from the input images.
+node {
+  calculator: "ImagePropertiesCalculator"
+  input_stream: "IMAGE:input_video"
+  output_stream: "SIZE:image_size"
+}
+
+# Gets ROI for re-crop model from face-related pose landmarks.
+node {
+  calculator: "FaceLandmarksFromPoseToRecropRoi"
+  input_stream: "FACE_LANDMARKS_FROM_POSE:face_landmarks_from_pose"
+  input_stream: "IMAGE_SIZE:image_size"
+  output_stream: "ROI:face_roi_from_pose"
+}
+
+# Detects faces within the face ROI calculated from pose landmarks. This is done
+# to refine face ROI for further landmark detection as ROI calculated from
+# pose landmarks may be inaccurate.
+node {
+  calculator: "FaceDetectionFrontByRoiCpu"
+  input_stream: "IMAGE:input_video"
+  input_stream: "ROI:face_roi_from_pose"
+  output_stream: "DETECTIONS:face_detections"
+}
+
+# Calculates refined face ROI.
+node {
+  calculator: "FaceDetectionFrontDetectionsToRoi",
+  input_stream: "DETECTIONS:face_detections"
+  input_stream: "IMAGE_SIZE:image_size"
+  output_stream: "ROI:face_roi_from_detection"
+}
+
+# Gets face tracking rectangle (either face rectangle from the previous
+# frame or face re-crop rectangle from the current frame) for face prediction.
+node {
+  calculator: "FaceTracking"
+  input_stream: "LANDMARKS:face_landmarks"
+  input_stream: "FACE_RECROP_ROI:face_roi_from_detection"
+  input_stream: "IMAGE_SIZE:image_size"
+  output_stream: "FACE_TRACKING_ROI:face_tracking_roi"
+}
+
+# Predicts face landmarks from the tracking rectangle.
+node {
+  calculator: "FaceLandmarkCpu"
+  input_stream: "IMAGE:input_video"
+  input_stream: "ROI:face_tracking_roi"
+  output_stream: "LANDMARKS:face_landmarks"
+}

mediapipe/modules/holistic_landmark/face_landmarks_from_pose_gpu.pbtxt

@@ -0,0 +1,76 @@
+# Predicts face landmarks within an ROI derived from face-related pose
+# landmarks.
+
+type: "FaceLandmarksFromPoseGpu"
+
+# GPU image. (ImageFrame)
+input_stream: "IMAGE:input_video"
+# Face-related pose landmarks. (NormalizedLandmarkList)
+input_stream: "FACE_LANDMARKS_FROM_POSE:face_landmarks_from_pose"
+
+# Face landmarks. (NormalizedLandmarkList)
+output_stream: "FACE_LANDMARKS:face_landmarks"
+
+# Debug outputs.
+# Face ROI derived from face-related pose landmarks, which defines the search
+# region for the face detection model. (NormalizedRect)
+output_stream: "FACE_ROI_FROM_POSE:face_roi_from_pose"
+# Refined face crop rectangle predicted by face detection model.
+# (NormalizedRect)
+output_stream: "FACE_ROI_FROM_DETECTION:face_roi_from_detection"
+# Rectangle used to predict face landmarks. (NormalizedRect)
+output_stream: "FACE_TRACKING_ROI:face_tracking_roi"
+
+# TODO: do not predict face when most of the face landmarks from
+# pose are invisible.
+
+# Extracts image size from the input images.
+node {
+  calculator: "ImagePropertiesCalculator"
+  input_stream: "IMAGE_GPU:input_video"
+  output_stream: "SIZE:image_size"
+}
+
+# Gets ROI for re-crop model from face-related pose landmarks.
+node {
+  calculator: "FaceLandmarksFromPoseToRecropRoi"
+  input_stream: "FACE_LANDMARKS_FROM_POSE:face_landmarks_from_pose"
+  input_stream: "IMAGE_SIZE:image_size"
+  output_stream: "ROI:face_roi_from_pose"
+}
+
+# Detects faces within the face ROI calculated from pose landmarks. This is done
+# to refine face ROI for further landmark detection as ROI calculated from
+# pose landmarks may be inaccurate.
+node {
+  calculator: "FaceDetectionFrontByRoiGpu"
+  input_stream: "IMAGE:input_video"
+  input_stream: "ROI:face_roi_from_pose"
+  output_stream: "DETECTIONS:face_detections"
+}
+
+# Calculates refined face ROI.
+node {
+  calculator: "FaceDetectionFrontDetectionsToRoi",
+  input_stream: "DETECTIONS:face_detections"
+  input_stream: "IMAGE_SIZE:image_size"
+  output_stream: "ROI:face_roi_from_detection"
+}
+
+# Gets face tracking rectangle (either face rectangle from the previous
+# frame or face re-crop rectangle from the current frame) for face prediction.
+node {
+  calculator: "FaceTracking"
+  input_stream: "LANDMARKS:face_landmarks"
+  input_stream: "FACE_RECROP_ROI:face_roi_from_detection"
+  input_stream: "IMAGE_SIZE:image_size"
+  output_stream: "FACE_TRACKING_ROI:face_tracking_roi"
+}
+
+# Predicts face landmarks from the tracking rectangle.
+node {
+  calculator: "FaceLandmarkGpu"
+  input_stream: "IMAGE:input_video"
+  input_stream: "ROI:face_tracking_roi"
+  output_stream: "LANDMARKS:face_landmarks"
+}

mediapipe/modules/holistic_landmark/face_landmarks_from_pose_to_recrop_roi.pbtxt

@@ -0,0 +1,51 @@
+# Converts face-related pose landmarks to re-crop ROI.
+
+type: "FaceLandmarksFromPoseToRecropRoi"
+
+# Face-related pose landmarks (There should be 11 of them).
+# (NormalizedLandmarkList)
+input_stream: "FACE_LANDMARKS_FROM_POSE:face_landmarks_from_pose"
+# Image size (width & height). (std::pair<int, int>)
+input_stream: "IMAGE_SIZE:image_size"
+
+# ROI to be used for face detection. (NormalizedRect)
+output_stream: "ROI:roi"
+
+# Converts face-related pose landmarks to a detection that tightly encloses all
+# landmarks.
+node {
+  calculator: "LandmarksToDetectionCalculator"
+  input_stream: "NORM_LANDMARKS:face_landmarks_from_pose"
+  output_stream: "DETECTION:pose_face_detection"
+}
+
+# Converts face detection to a normalized face rectangle.
+node {
+  calculator: "DetectionsToRectsCalculator"
+  input_stream: "DETECTION:pose_face_detection"
+  input_stream: "IMAGE_SIZE:image_size"
+  output_stream: "NORM_RECT:pose_face_rect"
+  options: {
+    [mediapipe.DetectionsToRectsCalculatorOptions.ext] {
+      rotation_vector_start_keypoint_index: 5  # Right eye.
+      rotation_vector_end_keypoint_index: 2  # Left eye.
+      rotation_vector_target_angle_degrees: 0
+    }
+  }
+}
+
+# Expands face rectangle so that it becomes big enough for face detector to
+# localize it accurately.
+node {
+  calculator: "RectTransformationCalculator"
+  input_stream: "NORM_RECT:pose_face_rect"
+  input_stream: "IMAGE_SIZE:image_size"
+  output_stream: "roi"
+  options: {
+    [mediapipe.RectTransformationCalculatorOptions.ext] {
+      scale_x: 3.0
+      scale_y: 3.0
+      square_long: true
+    }
+  }
+}

mediapipe/modules/holistic_landmark/face_landmarks_to_roi.pbtxt

@@ -0,0 +1,53 @@
+# Converts face landmarks to ROI.
+
+type: "FaceLandmarksToRoi"
+
+# Face landmarks. (NormalizedLandmarkList)
+input_stream: "LANDMARKS:face_landmarks"
+# Image size (width & height). (std::pair<int, int>)
+input_stream: "IMAGE_SIZE:image_size"
+
+# ROI according to landmarks. (NormalizedRect)
+output_stream: "ROI:roi"
+
+# Converts face landmarks to a detection that tightly encloses all landmarks.
+node {
+  calculator: "LandmarksToDetectionCalculator"
+  input_stream: "NORM_LANDMARKS:face_landmarks"
+  output_stream: "DETECTION:face_detection"
+}
+
+# Converts the face detection into a rectangle (normalized by image size)
+# that encloses the face and is rotated such that the line connecting center of
+# the wrist and MCP of the middle finger is aligned with the Y-axis of the
+# rectangle.
+node {
+  calculator: "DetectionsToRectsCalculator"
+  input_stream: "DETECTION:face_detection"
+  input_stream: "IMAGE_SIZE:image_size"
+  output_stream: "NORM_RECT:face_landmarks_rect_tight"
+  options: {
+    [mediapipe.DetectionsToRectsCalculatorOptions.ext] {
+      rotation_vector_start_keypoint_index: 33  # Right side of left eye.
+      rotation_vector_end_keypoint_index: 263  # Left side of right eye.
+      rotation_vector_target_angle_degrees: 0
+    }
+  }
+}
+
+# Expands the face rectangle so that it's likely to contain the face even with
+# some motion.
+node {
+  calculator: "RectTransformationCalculator"
+  input_stream: "NORM_RECT:face_landmarks_rect_tight"
+  input_stream: "IMAGE_SIZE:image_size"
+  output_stream: "roi"
+  options: {
+    [mediapipe.RectTransformationCalculatorOptions.ext] {
+      scale_x: 1.5
+      scale_y: 1.5
+      # TODO: remove `square_long` where appropriat
+      square_long: true
+    }
+  }
+}

mediapipe/modules/holistic_landmark/face_tracking.pbtxt

@@ -0,0 +1,61 @@
+# Decides what ROI to use for face landmarks prediction: either previous frame
+# landmarks ROI or the current frame face re-crop ROI.
+
+type: "FaceTracking"
+
+# Face landmarks from the current frame. They will be memorized for tracking on
+# the next frame. (NormalizedLandmarkList)
+input_stream: "LANDMARKS:face_landmarks"
+# Face re-crop ROI from the current frame. (NormalizedRect)
+input_stream: "FACE_RECROP_ROI:face_recrop_roi"
+# Image size (width & height). (std::pair<int, int>)
+input_stream: "IMAGE_SIZE:image_size"
+
+# Face tracking ROI. Which is either face landmarks ROI from the previous frame
+# if face is still tracked, or face re-crop ROI from the current frame
+# otherwise. (NormalizedRect)
+output_stream: "FACE_TRACKING_ROI:face_tracking_roi"
+
+# Keeps track of face landmarks from the previous frame.
+node {
+  calculator: "PreviousLoopbackCalculator"
+  input_stream: "MAIN:image_size"
+  input_stream: "LOOP:face_landmarks"
+  input_stream_info: {
+    tag_index: "LOOP"
+    back_edge: true
+  }
+  output_stream: "PREV_LOOP:prev_face_landmarks"
+}
+
+# Gets hand landarmsk rect.
+node {
+  calculator: "FaceLandmarksToRoi"
+  input_stream: "LANDMARKS:prev_face_landmarks"
+  input_stream: "IMAGE_SIZE:image_size"
+  output_stream: "ROI:prev_face_landmarks_rect"
+}
+
+# Checks that all requirements for tracking are satisfied and use face rectangle
+# from the previous frame in that case. Otherwise - use face re-crop rectangle
+# from the current frame.
+node {
+  calculator: "RoiTrackingCalculator"
+  input_stream: "PREV_LANDMARKS:prev_face_landmarks"
+  input_stream: "PREV_LANDMARKS_RECT:prev_face_landmarks_rect"
+  input_stream: "RECROP_RECT:face_recrop_roi"
+  input_stream: "IMAGE_SIZE:image_size"
+  output_stream: "TRACKING_RECT:face_tracking_roi"
+  options: {
+    [mediapipe.RoiTrackingCalculatorOptions.ext] {
+      rect_requirements: {
+        rotation_degrees: 15.0
+        translation: 0.1
+        scale: 0.3
+      }
+      landmarks_requirements: {
+        recrop_rect_margin: -0.2
+      }
+    }
+  }
+}

mediapipe/modules/holistic_landmark/hand_landmarks_from_pose_cpu.pbtxt

@@ -0,0 +1,78 @@
+# Predicts hand landmarks within a ROI derived from hand-related pose landmarks.
+
+type: "HandLandmarksFromPoseCpu"
+
+# CPU image. (ImageFrame)
+input_stream: "IMAGE:input_video"
+# Hand-related pose landmarks in [wrist, pinky, index] order.
+# (NormalizedLandmarkList)
+input_stream: "HAND_LANDMARKS_FROM_POSE:hand_landmarks_from_pose"
+
+# Hand landmarks. (NormalizedLandmarkList)
+output_stream: "HAND_LANDMARKS:hand_landmarks"
+
+# Debug outputs.
+# Hand ROI derived from hand-related landmarks, which defines the search region
+# for the hand re-crop model. (NormalizedRect)
+output_stream: "HAND_ROI_FROM_POSE:hand_roi_from_pose"
+# Refined hand crop rectangle predicted by hand re-crop model. (NormalizedRect)
+output_stream: "HAND_ROI_FROM_RECROP:hand_roi_from_recrop"
+# Rectangle used to predict hand landmarks. (NormalizedRect)
+output_stream: "HAND_TRACKING_ROI:hand_tracking_roi"
+
+# Gets hand visibility.
+node {
+  calculator: "HandVisibilityFromHandLandmarksFromPose"
+  input_stream: "HAND_LANDMARKS_FROM_POSE:hand_landmarks_from_pose"
+  output_stream: "VISIBILITY:hand_visibility"
+}
+
+# Drops hand-related pose landmarks if pose wrist is not visible. It will
+# prevent from predicting hand landmarks on the current frame.
+node {
+  calculator: "GateCalculator"
+  input_stream: "hand_landmarks_from_pose"
+  input_stream: "ALLOW:hand_visibility"
+  output_stream: "ensured_hand_landmarks_from_pose"
+}
+
+# Extracts image size from the input images.
+node {
+  calculator: "ImagePropertiesCalculator"
+  input_stream: "IMAGE:input_video"
+  output_stream: "SIZE:image_size"
+}
+
+# Gets ROI for re-crop model from hand-related pose landmarks.
+node {
+  calculator: "HandLandmarksFromPoseToRecropRoi"
+  input_stream: "HAND_LANDMARKS_FROM_POSE:ensured_hand_landmarks_from_pose"
+  input_stream: "IMAGE_SIZE:image_size"
+  output_stream: "ROI:hand_roi_from_pose"
+}
+
+# Predicts hand re-crop rectangle on the current frame.
+node {
+  calculator: "HandRecropByRoiCpu",
+  input_stream: "IMAGE:input_video"
+  input_stream: "ROI:hand_roi_from_pose"
+  output_stream: "HAND_ROI_FROM_RECROP:hand_roi_from_recrop"
+}
+
+# Gets hand tracking rectangle (either hand rectangle from the previous
+# frame or hand re-crop rectangle from the current frame) for hand prediction.
+node {
+  calculator: "HandTracking"
+  input_stream: "LANDMARKS:hand_landmarks"
+  input_stream: "HAND_ROI_FROM_RECROP:hand_roi_from_recrop"
+  input_stream: "IMAGE_SIZE:image_size"
+  output_stream: "HAND_TRACKING_ROI:hand_tracking_roi"
+}
+
+# Predicts hand landmarks from the tracking rectangle.
+node {
+  calculator: "HandLandmarkCpu"
+  input_stream: "IMAGE:input_video"
+  input_stream: "ROI:hand_tracking_roi"
+  output_stream: "LANDMARKS:hand_landmarks"
+}

mediapipe/modules/holistic_landmark/hand_landmarks_from_pose_gpu.pbtxt

@@ -0,0 +1,78 @@
+# Predicts hand landmarks within a ROI derived from hand-related pose landmarks.
+
+type: "HandLandmarksFromPoseGpu"
+
+# GPU image. (ImageFrame)
+input_stream: "IMAGE:input_video"
+# Hand-related pose landmarks in [wrist, pinky, index] order.
+# (NormalizedLandmarkList)
+input_stream: "HAND_LANDMARKS_FROM_POSE:hand_landmarks_from_pose"
+
+# Hand landmarks. (NormalizedLandmarkList)
+output_stream: "HAND_LANDMARKS:hand_landmarks"
+
+# Debug outputs.
+# Hand ROI derived from hand-related landmarks, which defines the search region
+# for the hand re-crop model. (NormalizedRect)
+output_stream: "HAND_ROI_FROM_POSE:hand_roi_from_pose"
+# Refined hand crop rectangle predicted by hand re-crop model. (NormalizedRect)
+output_stream: "HAND_ROI_FROM_RECROP:hand_roi_from_recrop"
+# Rectangle used to predict hand landmarks. (NormalizedRect)
+output_stream: "HAND_TRACKING_ROI:hand_tracking_roi"
+
+# Gets hand visibility.
+node {
+  calculator: "HandVisibilityFromHandLandmarksFromPose"
+  input_stream: "HAND_LANDMARKS_FROM_POSE:hand_landmarks_from_pose"
+  output_stream: "VISIBILITY:hand_visibility"
+}
+
+# Drops hand-related pose landmarks if pose wrist is not visible. It will
+# prevent from predicting hand landmarks on the current frame.
+node {
+  calculator: "GateCalculator"
+  input_stream: "hand_landmarks_from_pose"
+  input_stream: "ALLOW:hand_visibility"
+  output_stream: "ensured_hand_landmarks_from_pose"
+}
+
+# Extracts image size from the input images.
+node {
+  calculator: "ImagePropertiesCalculator"
+  input_stream: "IMAGE_GPU:input_video"
+  output_stream: "SIZE:image_size"
+}
+
+# Gets ROI for re-crop model from hand-related pose landmarks.
+node {
+  calculator: "HandLandmarksFromPoseToRecropRoi"
+  input_stream: "HAND_LANDMARKS_FROM_POSE:ensured_hand_landmarks_from_pose"
+  input_stream: "IMAGE_SIZE:image_size"
+  output_stream: "ROI:hand_roi_from_pose"
+}
+
+# Predicts hand re-crop rectangle on the current frame.
+node {
+  calculator: "HandRecropByRoiGpu",
+  input_stream: "IMAGE:input_video"
+  input_stream: "ROI:hand_roi_from_pose"
+  output_stream: "HAND_ROI_FROM_RECROP:hand_roi_from_recrop"
+}
+
+# Gets hand tracking rectangle (either hand rectangle from the previous
+# frame or hand re-crop rectangle from the current frame) for hand prediction.
+node {
+  calculator: "HandTracking"
+  input_stream: "LANDMARKS:hand_landmarks"
+  input_stream: "HAND_ROI_FROM_RECROP:hand_roi_from_recrop"
+  input_stream: "IMAGE_SIZE:image_size"
+  output_stream: "HAND_TRACKING_ROI:hand_tracking_roi"
+}
+
+# Predicts hand landmarks from the tracking rectangle.
+node {
+  calculator: "HandLandmarkGpu"
+  input_stream: "IMAGE:input_video"
+  input_stream: "ROI:hand_tracking_roi"
+  output_stream: "LANDMARKS:hand_landmarks"
+}

mediapipe/modules/holistic_landmark/hand_landmarks_from_pose_to_recrop_roi.pbtxt

@@ -0,0 +1,45 @@
+# Converts hand-related pose landmarks to hand re-crop ROI.
+
+type: "HandLandmarksFromPoseToRecropRoi"
+
+# Hand-related pose landmarks in [wrist, pinky, index] order.
+# (NormalizedLandmarkList)
+input_stream: "HAND_LANDMARKS_FROM_POSE:hand_landmarks_from_pose"
+# Image size (width & height). (std::pair<int, int>)
+input_stream: "IMAGE_SIZE:image_size"
+
+# ROI to be used for re-crop prediction. (NormalizedRect)
+output_stream: "ROI:roi"
+
+# Converts hand-related pose landmarks to a detection that tightly encloses all
+# of them.
+node {
+  calculator: "LandmarksToDetectionCalculator"
+  input_stream: "NORM_LANDMARKS:hand_landmarks_from_pose"
+  output_stream: "DETECTION:hand_detection_from_pose"
+}
+
+# Converts hand detection to a normalized hand rectangle.
+node {
+  calculator: "HandDetectionsFromPoseToRectsCalculator"
+  input_stream: "DETECTION:hand_detection_from_pose"
+  input_stream: "IMAGE_SIZE:image_size"
+  output_stream: "NORM_RECT:hand_roi_from_pose"
+}
+
+# Expands the palm rectangle so that it becomes big enough for hand re-crop
+# model to localize it accurately.
+node {
+  calculator: "RectTransformationCalculator"
+  input_stream: "NORM_RECT:hand_roi_from_pose"
+  input_stream: "IMAGE_SIZE:image_size"
+  output_stream: "roi"
+  options: {
+    [mediapipe.RectTransformationCalculatorOptions.ext] {
+      scale_x: 2.7
+      scale_y: 2.7
+      shift_y: -0.1
+      square_long: true
+    }
+  }
+}

mediapipe/modules/holistic_landmark/hand_landmarks_left_and_right_cpu.pbtxt

@@ -0,0 +1,76 @@
+# Predicts left and right hand landmarks within corresponding ROIs derived from
+# hand-related pose landmarks.
+
+type: "HandLandmarksLeftAndRightCpu"
+
+# CPU image. (ImageFrame)
+input_stream: "IMAGE:input_video"
+# Pose landmarks to derive initial hand location from. (NormalizedLandmarkList)
+input_stream: "POSE_LANDMARKS:pose_landmarks"
+
+# Left hand landmarks. (NormalizedLandmarkList)
+output_stream: "LEFT_HAND_LANDMARKS:left_hand_landmarks"
+# RIght hand landmarks. (NormalizedLandmarkList)
+output_stream: "RIGHT_HAND_LANDMARKS:right_hand_landmarks"
+
+# Debug outputs.
+output_stream: "LEFT_HAND_ROI_FROM_POSE:left_hand_roi_from_pose"
+output_stream: "LEFT_HAND_ROI_FROM_RECROP:left_hand_roi_from_recrop"
+output_stream: "LEFT_HAND_TRACKING_ROI:left_hand_tracking_roi"
+output_stream: "RIGHT_HAND_ROI_FROM_POSE:right_hand_roi_from_pose"
+output_stream: "RIGHT_HAND_ROI_FROM_RECROP:right_hand_roi_from_recrop"
+output_stream: "RIGHT_HAND_TRACKING_ROI:right_hand_tracking_roi"
+
+# Extracts left-hand-related landmarks from the pose landmarks.
+node {
+  calculator: "SplitNormalizedLandmarkListCalculator"
+  input_stream: "pose_landmarks"
+  output_stream: "left_hand_landmarks_from_pose"
+  options: {
+    [mediapipe.SplitVectorCalculatorOptions.ext] {
+      ranges: { begin: 15 end: 16 }
+      ranges: { begin: 17 end: 18 }
+      ranges: { begin: 19 end: 20 }
+      combine_outputs: true
+    }
+  }
+}
+
+# Predicts left hand landmarks.
+node {
+  calculator: "HandLandmarksFromPoseCpu"
+  input_stream: "IMAGE:input_video"
+  input_stream: "HAND_LANDMARKS_FROM_POSE:left_hand_landmarks_from_pose"
+  output_stream: "HAND_LANDMARKS:left_hand_landmarks"
+  # Debug outputs.
+  output_stream: "HAND_ROI_FROM_POSE:left_hand_roi_from_pose"
+  output_stream: "HAND_ROI_FROM_RECROP:left_hand_roi_from_recrop"
+  output_stream: "HAND_TRACKING_ROI:left_hand_tracking_roi"
+}
+
+# Extracts right-hand-related landmarks from the pose landmarks.
+node {
+  calculator: "SplitNormalizedLandmarkListCalculator"
+  input_stream: "pose_landmarks"
+  output_stream: "right_hand_landmarks_from_pose"
+  options: {
+    [mediapipe.SplitVectorCalculatorOptions.ext] {
+      ranges: { begin: 16 end: 17 }
+      ranges: { begin: 18 end: 19 }
+      ranges: { begin: 20 end: 21 }
+      combine_outputs: true
+    }
+  }
+}
+
+# Extracts right-hand-related landmarks from the pose landmarks.
+node {
+  calculator: "HandLandmarksFromPoseCpu"
+  input_stream: "IMAGE:input_video"
+  input_stream: "HAND_LANDMARKS_FROM_POSE:right_hand_landmarks_from_pose"
+  output_stream: "HAND_LANDMARKS:right_hand_landmarks"
+  # Debug outputs.
+  output_stream: "HAND_ROI_FROM_POSE:right_hand_roi_from_pose"
+  output_stream: "HAND_ROI_FROM_RECROP:right_hand_roi_from_recrop"
+  output_stream: "HAND_TRACKING_ROI:right_hand_tracking_roi"
+}

mediapipe/modules/holistic_landmark/hand_landmarks_left_and_right_gpu.pbtxt

@@ -0,0 +1,76 @@
+# Predicts left and right hand landmarks within corresponding ROIs derived from
+# hand-related pose landmarks.
+
+type: "HandLandmarksLeftAndRightGpu"
+
+# GPU image. (ImageFrame)
+input_stream: "IMAGE:input_video"
+# Pose landmarks to derive initial hand location from. (NormalizedLandmarkList)
+input_stream: "POSE_LANDMARKS:pose_landmarks"
+
+# Left hand landmarks. (NormalizedLandmarkList)
+output_stream: "LEFT_HAND_LANDMARKS:left_hand_landmarks"
+# RIght hand landmarks. (NormalizedLandmarkList)
+output_stream: "RIGHT_HAND_LANDMARKS:right_hand_landmarks"
+
+# Debug outputs.
+output_stream: "LEFT_HAND_ROI_FROM_POSE:left_hand_roi_from_pose"
+output_stream: "LEFT_HAND_ROI_FROM_RECROP:left_hand_roi_from_recrop"
+output_stream: "LEFT_HAND_TRACKING_ROI:left_hand_tracking_roi"
+output_stream: "RIGHT_HAND_ROI_FROM_POSE:right_hand_roi_from_pose"
+output_stream: "RIGHT_HAND_ROI_FROM_RECROP:right_hand_roi_from_recrop"
+output_stream: "RIGHT_HAND_TRACKING_ROI:right_hand_tracking_roi"
+
+# Extracts left-hand-related landmarks from the pose landmarks.
+node {
+  calculator: "SplitNormalizedLandmarkListCalculator"
+  input_stream: "pose_landmarks"
+  output_stream: "left_hand_landmarks_from_pose"
+  options: {
+    [mediapipe.SplitVectorCalculatorOptions.ext] {
+      ranges: { begin: 15 end: 16 }
+      ranges: { begin: 17 end: 18 }
+      ranges: { begin: 19 end: 20 }
+      combine_outputs: true
+    }
+  }
+}
+
+# Predicts left hand landmarks.
+node {
+  calculator: "HandLandmarksFromPoseGpu"
+  input_stream: "IMAGE:input_video"
+  input_stream: "HAND_LANDMARKS_FROM_POSE:left_hand_landmarks_from_pose"
+  output_stream: "HAND_LANDMARKS:left_hand_landmarks"
+  # Debug outputs.
+  output_stream: "HAND_ROI_FROM_POSE:left_hand_roi_from_pose"
+  output_stream: "HAND_ROI_FROM_RECROP:left_hand_roi_from_recrop"
+  output_stream: "HAND_TRACKING_ROI:left_hand_tracking_roi"
+}
+
+# Extracts right-hand-related landmarks from the pose landmarks.
+node {
+  calculator: "SplitNormalizedLandmarkListCalculator"
+  input_stream: "pose_landmarks"
+  output_stream: "right_hand_landmarks_from_pose"
+  options: {
+    [mediapipe.SplitVectorCalculatorOptions.ext] {
+      ranges: { begin: 16 end: 17 }
+      ranges: { begin: 18 end: 19 }
+      ranges: { begin: 20 end: 21 }
+      combine_outputs: true
+    }
+  }
+}
+
+# Extracts right-hand-related landmarks from the pose landmarks.
+node {
+  calculator: "HandLandmarksFromPoseGpu"
+  input_stream: "IMAGE:input_video"
+  input_stream: "HAND_LANDMARKS_FROM_POSE:right_hand_landmarks_from_pose"
+  output_stream: "HAND_LANDMARKS:right_hand_landmarks"
+  # Debug outputs.
+  output_stream: "HAND_ROI_FROM_POSE:right_hand_roi_from_pose"
+  output_stream: "HAND_ROI_FROM_RECROP:right_hand_roi_from_recrop"
+  output_stream: "HAND_TRACKING_ROI:right_hand_tracking_roi"
+}

mediapipe/modules/holistic_landmark/hand_landmarks_to_roi.pbtxt

@@ -0,0 +1,57 @@
+# Converts hand landmarks to ROI.
+
+type: "HandLandmarksToRoi"
+
+# Hand landmarks. (NormalizedLandmarkList)
+input_stream: "LANDMARKS:hand_landmarks"
+# Image size (width & height). (std::pair<int, int>)
+input_stream: "IMAGE_SIZE:image_size"
+
+# ROI according to the hand landmarks. (NormalizedRect)
+output_stream: "ROI:roi"
+
+# Gets hand palm landmarks.
+node {
+  calculator: "SplitNormalizedLandmarkListCalculator"
+  input_stream: "hand_landmarks"
+  output_stream: "palm_landmarks"
+  options: {
+    [mediapipe.SplitVectorCalculatorOptions.ext] {
+      ranges: { begin: 0 end: 4 }
+      ranges: { begin: 5 end: 7 }
+      ranges: { begin: 9 end: 11 }
+      ranges: { begin: 13 end: 15 }
+      ranges: { begin: 17 end: 19 }
+      combine_outputs: true
+    }
+  }
+}
+
+# Converts the hand landmarks into a rectangle (normalized by image size)
+# that encloses the hand. The calculator uses a subset of all hand landmarks
+# extracted from SplitNormalizedLandmarkListCalculator above to
+# calculate the bounding box and the rotation of the output rectangle. Please
+# see the comments in the calculator for more detail.
+node {
+  calculator: "HandLandmarksToRectCalculator"
+  input_stream: "NORM_LANDMARKS:palm_landmarks"
+  input_stream: "IMAGE_SIZE:image_size"
+  output_stream: "NORM_RECT:palm_landmarks_rect"
+}
+
+# Expands the hand rectangle so that it's likely to contain the hand even with
+# some motion.
+node {
+  calculator: "RectTransformationCalculator"
+  input_stream: "NORM_RECT:palm_landmarks_rect"
+  input_stream: "IMAGE_SIZE:image_size"
+  output_stream: "roi"
+  options: {
+    [mediapipe.RectTransformationCalculatorOptions.ext] {
+      scale_x: 2.0
+      scale_y: 2.0
+      shift_y: -0.1
+      square_long: true
+    }
+  }
+}

mediapipe/modules/holistic_landmark/hand_recrop_by_roi_cpu.pbtxt

@@ -0,0 +1,136 @@
+# Predicts more accurate hand location (re-crop ROI) within a given ROI.
+
+type: "HandRecropByRoiCpu"
+
+# CPU image. (ImageFrame)
+input_stream: "IMAGE:input_video"
+# ROI (region of interest) within the given image where a palm/hand is located.
+# (NormalizedRect)
+input_stream: "ROI:roi"
+
+# Refined (more accurate) ROI to use for hand landmark prediction.
+# (NormalizedRect)
+output_stream: "HAND_ROI_FROM_RECROP:hand_roi_from_recrop_refined"
+
+# Transforms hand ROI from the input image to a 256x256 tensor. Preserves aspect
+# ratio, which results in a letterbox padding.
+node {
+  calculator: "ImageToTensorCalculator"
+  input_stream: "IMAGE:input_video"
+  input_stream: "NORM_RECT:roi"
+  output_stream: "TENSORS:initial_crop_tensor"
+  output_stream: "LETTERBOX_PADDING:letterbox_padding"
+  options: {
+    [mediapipe.ImageToTensorCalculatorOptions.ext] {
+      output_tensor_width: 256
+      output_tensor_height: 256
+      keep_aspect_ratio: true
+      output_tensor_float_range {
+        min: 0.0
+        max: 1.0
+      }
+      # For OpenGL origin should be at the top left corner.
+      gpu_origin: TOP_LEFT,
+    }
+  }
+}
+
+# Predicts hand re-crop rectangle.
+node {
+  calculator: "InferenceCalculator"
+  input_stream: "TENSORS:initial_crop_tensor"
+  output_stream: "TENSORS:landmark_tensors"
+  options: {
+    [mediapipe.InferenceCalculatorOptions.ext] {
+      model_path: "mediapipe/modules/holistic_landmark/hand_recrop.tflite"
+    }
+  }
+}
+
+# Decodes the landmark tensors into a vector of landmarks, where the landmark
+# coordinates are normalized by the size of the input image to the model. Two
+# landmarks represent two virtual points: crop and scale of the new crop.
+node {
+  calculator: "TensorsToLandmarksCalculator"
+  input_stream: "TENSORS:landmark_tensors"
+  output_stream: "NORM_LANDMARKS:landmarks"
+  options: {
+    [mediapipe.TensorsToLandmarksCalculatorOptions.ext] {
+      num_landmarks: 2
+      input_image_width: 256
+      input_image_height: 256
+    }
+  }
+}
+
+# Adjusts landmarks (already normalized to [0.f, 1.f]) on the letterboxed hand
+# image (after image transformation with the FIT scale mode) to the
+# corresponding locations on the same image with the letterbox removed (hand
+# image before image transformation).
+node {
+  calculator: "LandmarkLetterboxRemovalCalculator"
+  input_stream: "LANDMARKS:landmarks"
+  input_stream: "LETTERBOX_PADDING:letterbox_padding"
+  output_stream: "LANDMARKS:scaled_landmarks"
+}
+
+# Projects the landmarks from the cropped hand image to the corresponding
+# locations on the full image before cropping (input to the graph).
+node {
+  calculator: "LandmarkProjectionCalculator"
+  input_stream: "NORM_LANDMARKS:scaled_landmarks"
+  input_stream: "NORM_RECT:roi"
+  output_stream: "NORM_LANDMARKS:alignment_landmarks"
+}
+
+# Converts hand landmarks to a detection that tightly encloses all landmarks.
+node {
+  calculator: "LandmarksToDetectionCalculator"
+  input_stream: "NORM_LANDMARKS:alignment_landmarks"
+  output_stream: "DETECTION:hand_detection"
+}
+
+# Extracts image size from the input images.
+node {
+  calculator: "ImagePropertiesCalculator"
+  input_stream: "IMAGE:input_video"
+  output_stream: "SIZE:image_size"
+}
+
+# Converts hand detection into a rectangle based on center and scale alignment
+# points.
+node {
+  calculator: "AlignmentPointsRectsCalculator"
+  input_stream: "DETECTION:hand_detection"
+  input_stream: "IMAGE_SIZE:image_size"
+  output_stream: "NORM_RECT:hand_roi_from_recrop"
+  options: {
+    [mediapipe.DetectionsToRectsCalculatorOptions.ext] {
+      rotation_vector_start_keypoint_index: 0
+      rotation_vector_end_keypoint_index: 1
+      rotation_vector_target_angle_degrees: -90
+    }
+  }
+}
+
+# TODO: revise hand recrop roi calculation.
+# Slighly moves hand re-crop rectangle from wrist towards fingertips. Due to the
+# new hand cropping logic, crop border is to close to finger tips while a lot of
+# space is below the wrist. And when moving hand up fast (with fingers pointing
+# up) and using hand rect from the previous frame for tracking - fingertips can
+# be cropped. This adjustment partially solves it, but hand cropping logic
+# should be reviewed.
+node {
+  calculator: "RectTransformationCalculator"
+  input_stream: "NORM_RECT:hand_roi_from_recrop"
+  input_stream: "IMAGE_SIZE:image_size"
+  output_stream: "hand_roi_from_recrop_refined"
+  options: {
+    [mediapipe.RectTransformationCalculatorOptions.ext] {
+      scale_x: 1.0
+      scale_y: 1.0
+      shift_y: -0.1
+      square_long: true
+    }
+  }
+}

mediapipe/modules/holistic_landmark/hand_recrop_by_roi_gpu.pbtxt

@@ -0,0 +1,136 @@
+# Predicts more accurate hand location (re-crop ROI) within a given ROI.
+
+type: "HandRecropByRoiGpu"
+
+# GPU image. (ImageFrame)
+input_stream: "IMAGE:input_video"
+# ROI (region of interest) within the given image where a palm/hand is located.
+# (NormalizedRect)
+input_stream: "ROI:roi"
+
+# Refined (more accurate) ROI to use for hand landmark prediction.
+# (NormalizedRect)
+output_stream: "HAND_ROI_FROM_RECROP:hand_roi_from_recrop_refined"
+
+# Transforms hand ROI from the input image to a 256x256 tensor. Preserves aspect
+# ratio, which results in a letterbox padding.
+node {
+  calculator: "ImageToTensorCalculator"
+  input_stream: "IMAGE_GPU:input_video"
+  input_stream: "NORM_RECT:roi"
+  output_stream: "TENSORS:initial_crop_tensor"
+  output_stream: "LETTERBOX_PADDING:letterbox_padding"
+  options: {
+    [mediapipe.ImageToTensorCalculatorOptions.ext] {
+      output_tensor_width: 256
+      output_tensor_height: 256
+      keep_aspect_ratio: true
+      output_tensor_float_range {
+        min: 0.0
+        max: 1.0
+      }
+      # For OpenGL origin should be at the top left corner.
+      gpu_origin: TOP_LEFT,
+    }
+  }
+}
+
+# Predicts hand re-crop rectangle.
+node {
+  calculator: "InferenceCalculator"
+  input_stream: "TENSORS:initial_crop_tensor"
+  output_stream: "TENSORS:landmark_tensors"
+  options: {
+    [mediapipe.InferenceCalculatorOptions.ext] {
+      model_path: "mediapipe/modules/holistic_landmark/hand_recrop.tflite"
+    }
+  }
+}
+
+# Decodes the landmark tensors into a vector of landmarks, where the landmark
+# coordinates are normalized by the size of the input image to the model. Two
+# landmarks represent two virtual points: crop and scale of the new crop.
+node {
+  calculator: "TensorsToLandmarksCalculator"
+  input_stream: "TENSORS:landmark_tensors"
+  output_stream: "NORM_LANDMARKS:landmarks"
+  options: {
+    [mediapipe.TensorsToLandmarksCalculatorOptions.ext] {
+      num_landmarks: 2
+      input_image_width: 256
+      input_image_height: 256
+    }
+  }
+}
+
+# Adjusts landmarks (already normalized to [0.f, 1.f]) on the letterboxed hand
+# image (after image transformation with the FIT scale mode) to the
+# corresponding locations on the same image with the letterbox removed (hand
+# image before image transformation).
+node {
+  calculator: "LandmarkLetterboxRemovalCalculator"
+  input_stream: "LANDMARKS:landmarks"
+  input_stream: "LETTERBOX_PADDING:letterbox_padding"
+  output_stream: "LANDMARKS:scaled_landmarks"
+}
+
+# Projects the landmarks from the cropped hand image to the corresponding
+# locations on the full image before cropping (input to the graph).
+node {
+  calculator: "LandmarkProjectionCalculator"
+  input_stream: "NORM_LANDMARKS:scaled_landmarks"
+  input_stream: "NORM_RECT:roi"
+  output_stream: "NORM_LANDMARKS:alignment_landmarks"
+}
+
+# Converts hand landmarks to a detection that tightly encloses all landmarks.
+node {
+  calculator: "LandmarksToDetectionCalculator"
+  input_stream: "NORM_LANDMARKS:alignment_landmarks"
+  output_stream: "DETECTION:hand_detection"
+}
+
+# Extracts image size from the input images.
+node {
+  calculator: "ImagePropertiesCalculator"
+  input_stream: "IMAGE_GPU:input_video"
+  output_stream: "SIZE:image_size"
+}
+
+# Converts hand detection into a rectangle based on center and scale alignment
+# points.
+node {
+  calculator: "AlignmentPointsRectsCalculator"
+  input_stream: "DETECTION:hand_detection"
+  input_stream: "IMAGE_SIZE:image_size"
+  output_stream: "NORM_RECT:hand_roi_from_recrop"
+  options: {
+    [mediapipe.DetectionsToRectsCalculatorOptions.ext] {
+      rotation_vector_start_keypoint_index: 0
+      rotation_vector_end_keypoint_index: 1
+      rotation_vector_target_angle_degrees: -90
+    }
+  }
+}
+
+# TODO: revise hand recrop roi calculation.
+# Slighly moves hand re-crop rectangle from wrist towards fingertips. Due to the
+# new hand cropping logic, crop border is to close to finger tips while a lot of
+# space is below the wrist. And when moving hand up fast (with fingers pointing
+# up) and using hand rect from the previous frame for tracking - fingertips can
+# be cropped. This adjustment partially solves it, but hand cropping logic
+# should be reviewed.
+node {
+  calculator: "RectTransformationCalculator"
+  input_stream: "NORM_RECT:hand_roi_from_recrop"
+  input_stream: "IMAGE_SIZE:image_size"
+  output_stream: "hand_roi_from_recrop_refined"
+  options: {
+    [mediapipe.RectTransformationCalculatorOptions.ext] {
+      scale_x: 1.0
+      scale_y: 1.0
+      shift_y: -0.1
+      square_long: true
+    }
+  }
+}

mediapipe/modules/holistic_landmark/hand_tracking.pbtxt

@@ -0,0 +1,63 @@
+# Decides what ROI to use for hand landmark prediction: either previous frame
+# landmarks ROI or current frame re-crop ROI.
+
+type: "HandTracking"
+
+# Hand landmarks from the current frame. They will be memorized for tracking on
+# the next frame. (NormalizedLandmarkList)
+input_stream: "LANDMARKS:hand_landmarks"
+# Hand re-crop ROI from the current frame. (NormalizedRect)
+input_stream: "HAND_ROI_FROM_RECROP:hand_roi_from_recrop"
+# Image size (width & height). (std::pair<int, int>)
+input_stream: "IMAGE_SIZE:image_size"
+
+# Hand tracking ROI. Which is either hand landmarks ROI from the previous frame
+# if hand is still tracked, or hand re-crop ROI from the current frame
+# othervise. (NormalizedRect)
+output_stream: "HAND_TRACKING_ROI:hand_tracking_roi"
+
+# Keeps track of hand landmarks from the previous frame.
+node {
+  calculator: "PreviousLoopbackCalculator"
+  # TODO: check that loop works with image size instead of video.
+  input_stream: "MAIN:image_size"
+  input_stream: "LOOP:hand_landmarks"
+  input_stream_info: {
+    tag_index: "LOOP"
+    back_edge: true
+  }
+  output_stream: "PREV_LOOP:prev_hand_landmarks"
+}
+
+# Gets hand landarmsk rect.
+node {
+  calculator: "HandLandmarksToRoi"
+  input_stream: "LANDMARKS:prev_hand_landmarks"
+  input_stream: "IMAGE_SIZE:image_size"
+  output_stream: "ROI:prev_hand_landmarks_roi"
+}
+
+# Checks that all requirements for tracking are satisfied and use hand rectangle
+# from the previous frame in that case. Otherwise - use hand re-crop rectangle
+# from the current frame.
+node {
+  calculator: "RoiTrackingCalculator"
+  input_stream: "PREV_LANDMARKS:prev_hand_landmarks"
+  input_stream: "PREV_LANDMARKS_RECT:prev_hand_landmarks_roi"
+  input_stream: "RECROP_RECT:hand_roi_from_recrop"
+  input_stream: "IMAGE_SIZE:image_size"
+  output_stream: "TRACKING_RECT:hand_tracking_roi"
+  options: {
+    [mediapipe.RoiTrackingCalculatorOptions.ext] {
+      rect_requirements: {
+        rotation_degrees: 40.0
+        translation: 0.2
+        # TODO: adjust scale for hand tracking.
+        scale: 0.4
+      }
+      landmarks_requirements: {
+        recrop_rect_margin: -0.1
+      }
+    }
+  }
+}

mediapipe/modules/holistic_landmark/hand_visibility_from_hand_landmarks_from_pose.pbtxt

@@ -0,0 +1,44 @@
+# Determines hand visibility from the visibility prediction values in the
+# hand-related pose landmarks.
+
+type: "HandVisibilityFromHandLandmarksFromPose"
+
+# Hand-related pose landmarks in [wrist, pinky, index] order.
+# (NormalizedLandmarkList)
+input_stream: "HAND_LANDMARKS_FROM_POSE:hand_landmarks_from_pose"
+
+# Hand visibility to be used as a trigger for hand landmark prediction. (bool)
+output_stream: "VISIBILITY:wrist_visibility"
+
+# Gets pose wrist landmark.
+node {
+  calculator: "SplitNormalizedLandmarkListCalculator"
+  input_stream: "hand_landmarks_from_pose"
+  output_stream: "pose_wrist_landmark"
+  options: {
+    [mediapipe.SplitVectorCalculatorOptions.ext] {
+      ranges: { begin: 0 end: 1 }
+    }
+  }
+}
+
+# TODO: Use other than pose wrist palm landmarks.
+# Gets pose wrist visiblity.
+node {
+  calculator: "LandmarkVisibilityCalculator"
+  input_stream: "NORM_LANDMARKS:pose_wrist_landmark"
+  output_stream: "VISIBILITY:wrist_visibility_score"
+}
+
+# TODO: ensure the same threshold in rendering.
+# Converts pose wrist visibility score into boolean flag.
+node {
+  calculator: "ThresholdingCalculator"
+  input_stream: "FLOAT:wrist_visibility_score"
+  output_stream: "FLAG:wrist_visibility"
+  options: {
+    [mediapipe.ThresholdingCalculatorOptions.ext] {
+      threshold: 0.1
+    }
+  }
+}

mediapipe/modules/holistic_landmark/hand_wrist_for_pose.pbtxt

@@ -0,0 +1,52 @@
+# Extracts hand wrist landmark to be used instead of pose wrist landmark.
+
+type: "HandWristForPose"
+
+# Hand landmarks to take wrist landmark from. (NormalizedLandmarkList)
+input_stream: "HAND_LANDMARKS:hand_landmarks"
+
+# Hand wrist landmark to replace original pose wrist landmark with updated
+# visibility. (NormalizedLandmarkList)
+output_stream: "WRIST_LANDMARK:hand_wrist_landmark_with_visibility"
+
+# Side packet with constant for visibility score. As score is `x` from
+# `sigmoid(x)` we pick some big value that doesn't affect pose landmarks
+# visibility rendering threshold.
+node {
+  calculator: "ConstantSidePacketCalculator"
+  output_side_packet: "PACKET:0:visible_score_side_packet"
+  options: {
+    [mediapipe.ConstantSidePacketCalculatorOptions.ext]: {
+      packet { float_value: 100.0 }
+    }
+  }
+}
+
+# Converts side packet with visibility score to a stream.
+node {
+  calculator: "SidePacketToStreamCalculator"
+  input_stream: "TICK:hand_landmarks"
+  input_side_packet: "visible_score_side_packet"
+  output_stream: "AT_TICK:visible_score"
+}
+
+# Extracts wrist landmark from the hand landmarks.
+node {
+  calculator: "SplitNormalizedLandmarkListCalculator"
+  input_stream: "hand_landmarks"
+  output_stream: "hand_wrist_landmark"
+  options: {
+    [mediapipe.SplitVectorCalculatorOptions.ext] {
+      ranges: { begin: 0 end: 1 }
+    }
+  }
+}
+
+# Sets wrist landmark visibility score. If HAND_LANDMARKS is non-empty - wrist
+# will always be visible.
+node {
+  calculator: "SetLandmarkVisibilityCalculator"
+  input_stream: "NORM_LANDMARKS:hand_wrist_landmark"
+  input_stream: "VISIBILITY:visible_score"
+  output_stream: "NORM_LANDMARKS:hand_wrist_landmark_with_visibility"
+}

mediapipe/modules/holistic_landmark/holistic_landmark_cpu.pbtxt

@@ -0,0 +1,114 @@
+# Predicts pose + left/right hand + face landmarks.
+#
+# It is required that:
+# - "face_detection_front.tflite" is available at
+# "mediapipe/modules/face_detection/face_detection_front.tflite"
+#
+# - "face_landmark.tflite" is available at
+# "mediapipe/modules/face_landmark/face_landmark.tflite"
+#
+# - "hand_landmark.tflite" is available at
+# "mediapipe/modules/hand_landmark/hand_landmark.tflite"
+#
+# - "hand_recrop.tflite" is available at
+# "mediapipe/modules/holistic_landmark/hand_recrop.tflite"
+#
+# - "handedness.txt" is available at
+# "mediapipe/modules/hand_landmark/handedness.txt"
+#
+# - "pose_detection.tflite" is available at
+# "mediapipe/modules/pose_detection/pose_detection.tflite"
+#
+# - "pose_landmark_full_body.tflite" or "pose_landmark_upper_body.tflite" is
+# available at
+# "mediapipe/modules/pose_landmark/pose_landmark_full_body.tflite"
+# or
+# "mediapipe/modules/pose_landmark/pose_landmark_upper_body.tflite"
+# path respectively during execution, depending on the specification in the
+# UPPER_BODY_ONLY input side packet.
+#
+# EXAMPLE:
+#   node {
+#     calculator: "HolisticLandmarkCpu"
+#     input_stream: "IMAGE:input_video"
+#     input_side_packet: UPPER_BODY_ONLY:upper_body_only
+#     input_side_packet: SMOOTH_LANDMARKS:smooth_landmarks
+#     output_stream: "POSE_LANDMARKS:pose_landmarks"
+#     output_stream: "FACE_LANDMARKS:face_landmarks"
+#     output_stream: "LEFT_HAND_LANDMARKS:left_hand_landmarks"
+#     output_stream: "RIGHT_HAND_LANDMARKS:right_hand_landmarks"
+#   }
+#
+# NOTE: if a pose/hand/face output is not present in the image, for this
+# particular timestamp there will not be an output packet in the corresponding
+# output stream below. However, the MediaPipe framework will internally inform
+# the downstream calculators of the absence of this packet so that they don't
+# wait for it unnecessarily.
+
+type: "HolisticLandmarkCpu"
+
+# CPU image. (ImageFrame)
+input_stream: "IMAGE:image"
+
+# Whether to detect/predict the full set of pose landmarks (see below), or only
+# those on the upper body. If unspecified, functions as set to false. (bool)
+# Note that upper-body-only prediction may be more accurate for use cases where
+# the lower-body parts are mostly out of view.
+input_side_packet: "UPPER_BODY_ONLY:upper_body_only"
+# Whether to filter landmarks across different input images to reduce jitter.
+# If unspecified, functions as set to true. (bool)
+input_side_packet: "SMOOTH_LANDMARKS:smooth_landmarks"
+
+# Pose landmarks. (NormalizedLandmarkList)
+# We have 33 landmarks or 25 landmarks if UPPER_BODY_ONLY is set to true.
+output_stream: "POSE_LANDMARKS:pose_landmarks"
+# 21 left hand landmarks. (NormalizedLandmarkList)
+output_stream: "LEFT_HAND_LANDMARKS:left_hand_landmarks"
+# 21 right hand landmarks. (NormalizedLandmarkList)
+output_stream: "RIGHT_HAND_LANDMARKS:right_hand_landmarks"
+# 468 face landmarks. (NormalizedLandmarkList)
+output_stream: "FACE_LANDMARKS:face_landmarks"
+
+# Debug outputs
+output_stream: "POSE_ROI:pose_landmarks_roi"
+output_stream: "POSE_DETECTION:pose_detection"
+
+# Predicts pose landmarks.
+node {
+  calculator: "PoseLandmarkCpu"
+  input_stream: "IMAGE:image"
+  input_side_packet: "UPPER_BODY_ONLY:upper_body_only"
+  input_side_packet: "SMOOTH_LANDMARKS:smooth_landmarks"
+  output_stream: "LANDMARKS:pose_landmarks"
+  output_stream: "ROI_FROM_LANDMARKS:pose_landmarks_roi"
+  output_stream: "DETECTION:pose_detection"
+}
+
+# Predicts left and right hand landmarks based on the initial pose landmarks.
+node {
+  calculator: "HandLandmarksLeftAndRightCpu"
+  input_stream: "IMAGE:image"
+  input_stream: "POSE_LANDMARKS:pose_landmarks"
+  output_stream: "LEFT_HAND_LANDMARKS:left_hand_landmarks"
+  output_stream: "RIGHT_HAND_LANDMARKS:right_hand_landmarks"
+}
+
+# Extracts face-related pose landmarks.
+node {
+  calculator: "SplitNormalizedLandmarkListCalculator"
+  input_stream: "pose_landmarks"
+  output_stream: "face_landmarks_from_pose"
+  options: {
+    [mediapipe.SplitVectorCalculatorOptions.ext] {
+      ranges: { begin: 0 end: 11 }
+    }
+  }
+}
+
+# Predicts face landmarks based on the initial pose landmarks.
+node {
+  calculator: "FaceLandmarksFromPoseCpu"
+  input_stream: "IMAGE:image"
+  input_stream: "FACE_LANDMARKS_FROM_POSE:face_landmarks_from_pose"
+  output_stream: "FACE_LANDMARKS:face_landmarks"
+}

mediapipe/modules/holistic_landmark/holistic_landmark_gpu.pbtxt

@@ -0,0 +1,114 @@
+# Predicts pose + left/right hand + face landmarks.
+#
+# It is required that:
+# - "face_detection_front.tflite" is available at
+# "mediapipe/modules/face_detection/face_detection_front.tflite"
+#
+# - "face_landmark.tflite" is available at
+# "mediapipe/modules/face_landmark/face_landmark.tflite"
+#
+# - "hand_landmark.tflite" is available at
+# "mediapipe/modules/hand_landmark/hand_landmark.tflite"
+#
+# - "hand_recrop.tflite" is available at
+# "mediapipe/modules/holistic_landmark/hand_recrop.tflite"
+#
+# - "handedness.txt" is available at
+# "mediapipe/modules/hand_landmark/handedness.txt"
+#
+# - "pose_detection.tflite" is available at
+# "mediapipe/modules/pose_detection/pose_detection.tflite"
+#
+# - "pose_landmark_full_body.tflite" or "pose_landmark_upper_body.tflite" is
+# available at
+# "mediapipe/modules/pose_landmark/pose_landmark_full_body.tflite"
+# or
+# "mediapipe/modules/pose_landmark/pose_landmark_upper_body.tflite"
+# path respectively during execution, depending on the specification in the
+# UPPER_BODY_ONLY input side packet.
+#
+# EXAMPLE:
+#   node {
+#     calculator: "HolisticLandmarkGpu"
+#     input_stream: "IMAGE:input_video"
+#     input_side_packet: UPPER_BODY_ONLY:upper_body_only
+#     input_side_packet: SMOOTH_LANDMARKS:smooth_landmarks
+#     output_stream: "POSE_LANDMARKS:pose_landmarks"
+#     output_stream: "FACE_LANDMARKS:face_landmarks"
+#     output_stream: "LEFT_HAND_LANDMARKS:left_hand_landmarks"
+#     output_stream: "RIGHT_HAND_LANDMARKS:right_hand_landmarks"
+#   }
+#
+# NOTE: if a pose/hand/face output is not present in the image, for this
+# particular timestamp there will not be an output packet in the corresponding
+# output stream below. However, the MediaPipe framework will internally inform
+# the downstream calculators of the absence of this packet so that they don't
+# wait for it unnecessarily.
+
+type: "HolisticLandmarkGpu"
+
+# GPU image. (GpuBuffer)
+input_stream: "IMAGE:image"
+
+# Whether to detect/predict the full set of pose landmarks (see below), or only
+# those on the upper body. If unspecified, functions as set to false. (bool)
+# Note that upper-body-only prediction may be more accurate for use cases where
+# the lower-body parts are mostly out of view.
+input_side_packet: "UPPER_BODY_ONLY:upper_body_only"
+# Whether to filter landmarks across different input images to reduce jitter.
+# If unspecified, functions as set to true. (bool)
+input_side_packet: "SMOOTH_LANDMARKS:smooth_landmarks"
+
+# Pose landmarks. (NormalizedLandmarkList)
+# We have 33 landmarks or 25 landmarks if UPPER_BODY_ONLY is set to true.
+output_stream: "POSE_LANDMARKS:pose_landmarks"
+# 21 left hand landmarks. (NormalizedLandmarkList)
+output_stream: "LEFT_HAND_LANDMARKS:left_hand_landmarks"
+# 21 right hand landmarks. (NormalizedLandmarkList)
+output_stream: "RIGHT_HAND_LANDMARKS:right_hand_landmarks"
+# 468 face landmarks. (NormalizedLandmarkList)
+output_stream: "FACE_LANDMARKS:face_landmarks"
+
+# Debug outputs
+output_stream: "POSE_ROI:pose_landmarks_roi"
+output_stream: "POSE_DETECTION:pose_detection"
+
+# Predicts pose landmarks.
+node {
+  calculator: "PoseLandmarkGpu"
+  input_stream: "IMAGE:image"
+  input_side_packet: "UPPER_BODY_ONLY:upper_body_only"
+  input_side_packet: "SMOOTH_LANDMARKS:smooth_landmarks"
+  output_stream: "LANDMARKS:pose_landmarks"
+  output_stream: "ROI_FROM_LANDMARKS:pose_landmarks_roi"
+  output_stream: "DETECTION:pose_detection"
+}
+
+# Predicts left and right hand landmarks based on the initial pose landmarks.
+node {
+  calculator: "HandLandmarksLeftAndRightGpu"
+  input_stream: "IMAGE:image"
+  input_stream: "POSE_LANDMARKS:pose_landmarks"
+  output_stream: "LEFT_HAND_LANDMARKS:left_hand_landmarks"
+  output_stream: "RIGHT_HAND_LANDMARKS:right_hand_landmarks"
+}
+
+# Extracts face-related pose landmarks.
+node {
+  calculator: "SplitNormalizedLandmarkListCalculator"
+  input_stream: "pose_landmarks"
+  output_stream: "face_landmarks_from_pose"
+  options: {
+    [mediapipe.SplitVectorCalculatorOptions.ext] {
+      ranges: { begin: 0 end: 11 }
+    }
+  }
+}
+
+# Predicts face landmarks based on the initial pose landmarks.
+node {
+  calculator: "FaceLandmarksFromPoseGpu"
+  input_stream: "IMAGE:image"
+  input_stream: "FACE_LANDMARKS_FROM_POSE:face_landmarks_from_pose"
+  output_stream: "FACE_LANDMARKS:face_landmarks"
+}

mediapipe/modules/objectron/BUILD

@@ -0,0 +1,168 @@
+# Copyright 2020 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load(
+    "//mediapipe/framework/tool:mediapipe_graph.bzl",
+    "mediapipe_simple_subgraph",
+)
+
+licenses(["notice"])
+
+package(default_visibility = ["//visibility:public"])
+
+mediapipe_simple_subgraph(
+    name = "objectron_detection_1stage_gpu",
+    graph = "objectron_detection_1stage_gpu.pbtxt",
+    register_as = "ObjectronDetection1StageSubgraphGpu",
+    deps = [
+        "//mediapipe/calculators/image:image_transformation_calculator",
+        "//mediapipe/calculators/tflite:tflite_converter_calculator",
+        "//mediapipe/calculators/tflite:tflite_custom_op_resolver_calculator",
+        "//mediapipe/calculators/tflite:tflite_inference_calculator",
+        "//mediapipe/modules/objectron/calculators:tflite_tensors_to_objects_calculator",
+    ],
+)
+
+mediapipe_simple_subgraph(
+    name = "objectron_tracking_1stage_gpu",
+    graph = "objectron_tracking_1stage_gpu.pbtxt",
+    register_as = "ObjectronTracking1StageSubgraphGpu",
+    deps = [
+        "//mediapipe/calculators/image:image_transformation_calculator",
+        "//mediapipe/calculators/video:box_tracker_calculator",
+        "//mediapipe/calculators/video:flow_packager_calculator",
+        "//mediapipe/calculators/video:motion_analysis_calculator",
+        "//mediapipe/framework/stream_handler:sync_set_input_stream_handler",
+        "//mediapipe/gpu:gpu_buffer_to_image_frame_calculator",
+        "//mediapipe/modules/objectron/calculators:frame_annotation_to_timed_box_list_calculator",
+        "//mediapipe/modules/objectron/calculators:frame_annotation_tracker_calculator",
+        "//mediapipe/modules/objectron/calculators:lift_2d_frame_annotation_to_3d_calculator",
+    ],
+)
+
+mediapipe_simple_subgraph(
+    name = "box_landmark_gpu",
+    graph = "box_landmark_gpu.pbtxt",
+    register_as = "BoxLandmarkSubgraph",
+    deps = [
+        "//mediapipe/calculators/core:split_vector_calculator",
+        "//mediapipe/calculators/image:image_cropping_calculator",
+        "//mediapipe/calculators/image:image_properties_calculator",
+        "//mediapipe/calculators/image:image_transformation_calculator",
+        "//mediapipe/calculators/tflite:tflite_converter_calculator",
+        "//mediapipe/calculators/tflite:tflite_custom_op_resolver_calculator",
+        "//mediapipe/calculators/tflite:tflite_inference_calculator",
+        "//mediapipe/calculators/tflite:tflite_tensors_to_floats_calculator",
+        "//mediapipe/calculators/tflite:tflite_tensors_to_landmarks_calculator",
+        "//mediapipe/calculators/util:detections_to_rects_calculator",
+        "//mediapipe/calculators/util:landmark_letterbox_removal_calculator",
+        "//mediapipe/calculators/util:landmark_projection_calculator",
+        "//mediapipe/calculators/util:landmarks_smoothing_calculator",
+        "//mediapipe/calculators/util:landmarks_to_detection_calculator",
+        "//mediapipe/calculators/util:rect_transformation_calculator",
+        "//mediapipe/calculators/util:thresholding_calculator",
+        "//mediapipe/modules/objectron/calculators:frame_annotation_to_rect_calculator",
+        "//mediapipe/modules/objectron/calculators:landmarks_to_frame_annotation_calculator",
+        "//mediapipe/modules/objectron/calculators:lift_2d_frame_annotation_to_3d_calculator",
+    ],
+)
+
+mediapipe_simple_subgraph(
+    name = "box_landmark_cpu",
+    graph = "box_landmark_cpu.pbtxt",
+    register_as = "BoxLandmarkSubgraph",
+    deps = [
+        "//mediapipe/calculators/core:split_vector_calculator",
+        "//mediapipe/calculators/image:image_cropping_calculator",
+        "//mediapipe/calculators/image:image_properties_calculator",
+        "//mediapipe/calculators/image:image_transformation_calculator",
+        "//mediapipe/calculators/tflite:tflite_converter_calculator",
+        "//mediapipe/calculators/tflite:tflite_custom_op_resolver_calculator",
+        "//mediapipe/calculators/tflite:tflite_inference_calculator",
+        "//mediapipe/calculators/tflite:tflite_tensors_to_floats_calculator",
+        "//mediapipe/calculators/tflite:tflite_tensors_to_landmarks_calculator",
+        "//mediapipe/calculators/util:landmark_letterbox_removal_calculator",
+        "//mediapipe/calculators/util:landmark_projection_calculator",
+        "//mediapipe/calculators/util:landmarks_smoothing_calculator",
+        "//mediapipe/calculators/util:landmarks_to_detection_calculator",
+        "//mediapipe/calculators/util:rect_transformation_calculator",
+        "//mediapipe/calculators/util:thresholding_calculator",
+        "//mediapipe/modules/objectron/calculators:frame_annotation_to_rect_calculator",
+        "//mediapipe/modules/objectron/calculators:landmarks_to_frame_annotation_calculator",
+        "//mediapipe/modules/objectron/calculators:lift_2d_frame_annotation_to_3d_calculator",
+    ],
+)
+
+mediapipe_simple_subgraph(
+    name = "object_detection_oid_v4_gpu",
+    graph = "object_detection_oid_v4_gpu.pbtxt",
+    register_as = "ObjectDetectionOidV4Subgraph",
+    deps = [
+        "//mediapipe/calculators/image:image_transformation_calculator",
+        "//mediapipe/calculators/tflite:ssd_anchors_calculator",
+        "//mediapipe/calculators/tflite:tflite_converter_calculator",
+        "//mediapipe/calculators/tflite:tflite_inference_calculator",
+        "//mediapipe/calculators/tflite:tflite_tensors_to_detections_calculator",
+        "//mediapipe/calculators/util:detection_label_id_to_text_calculator",
+        "//mediapipe/calculators/util:detections_to_rects_calculator",
+        "//mediapipe/calculators/util:non_max_suppression_calculator",
+        "//mediapipe/calculators/util:rect_transformation_calculator",
+        "//mediapipe/modules/objectron/calculators:filter_detection_calculator",
+    ],
+)
+
+mediapipe_simple_subgraph(
+    name = "object_detection_oid_v4_cpu",
+    graph = "object_detection_oid_v4_cpu.pbtxt",
+    register_as = "ObjectDetectionOidV4Subgraph",
+    deps = [
+        "//mediapipe/calculators/image:image_transformation_calculator",
+        "//mediapipe/calculators/tflite:ssd_anchors_calculator",
+        "//mediapipe/calculators/tflite:tflite_converter_calculator",
+        "//mediapipe/calculators/tflite:tflite_inference_calculator",
+        "//mediapipe/calculators/tflite:tflite_tensors_to_detections_calculator",
+        "//mediapipe/calculators/util:detection_label_id_to_text_calculator",
+        "//mediapipe/calculators/util:detections_to_rects_calculator",
+        "//mediapipe/calculators/util:non_max_suppression_calculator",
+        "//mediapipe/calculators/util:rect_transformation_calculator",
+        "//mediapipe/modules/objectron/calculators:filter_detection_calculator",
+    ],
+)
+
+mediapipe_simple_subgraph(
+    name = "objectron_cpu",
+    graph = "objectron_cpu.pbtxt",
+    register_as = "ObjectronCpuSubgraph",
+    deps = [
+        ":box_landmark_cpu",
+        ":object_detection_oid_v4_cpu",
+        "//mediapipe/calculators/core:gate_calculator",
+        "//mediapipe/calculators/core:merge_calculator",
+        "//mediapipe/calculators/core:previous_loopback_calculator",
+    ],
+)
+
+mediapipe_simple_subgraph(
+    name = "objectron_gpu",
+    graph = "objectron_gpu.pbtxt",
+    register_as = "ObjectronGpuSubgraph",
+    deps = [
+        ":box_landmark_gpu",
+        ":object_detection_oid_v4_gpu",
+        "//mediapipe/calculators/core:gate_calculator",
+        "//mediapipe/calculators/core:merge_calculator",
+        "//mediapipe/calculators/core:previous_loopback_calculator",
+        "//mediapipe/calculators/image:image_cropping_calculator",
+    ],
+)

mediapipe/modules/objectron/README.md

@@ -0,0 +1,6 @@
+# objectron
+
+Subgraphs|Details
+:--- | :---
+[`ObjectronCpuSubgraph`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/objectron/objectron_cpu.pbtxt)| Detects and tracks 3D bounding boxes for objects. (CPU input, and inference is executed on CPU.)
+[`ObjectronGpuSubgraph`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/objectron/objectron_gpu.pbtxt)| Detects and tracks 3D bounding boxes for objects. (GPU input, and inference is executed on GPU.)

mediapipe/modules/objectron/box_landmark_cpu.pbtxt

@@ -0,0 +1,206 @@
+# MediaPipe Box landmark localization CPU subgraph.
+
+type: "BoxLandmarkSubgraph"
+
+input_stream: "IMAGE:input_video"
+input_stream: "NORM_RECT:box_rect"
+input_side_packet: "MODEL:model"
+output_stream: "LANDMARKS:box_landmarks_filtered"
+output_stream: "NORM_RECT:box_rect_for_next_frame"
+output_stream: "PRESENCE:box_presence"
+
+# Crops the rectangle that contains a box from the input image.
+node {
+  calculator: "ImageCroppingCalculator"
+  input_stream: "IMAGE:input_video"
+  input_stream: "NORM_RECT:box_rect"
+  output_stream: "IMAGE:box_image"
+  options: {
+    [mediapipe.ImageCroppingCalculatorOptions.ext] {
+      border_mode: BORDER_REPLICATE
+    }
+  }
+}
+
+# Transforms the input image to a 224x224 image. To scale the input
+# image, the scale_mode option is set to FIT to preserve the aspect ratio,
+# resulting in potential letterboxing in the transformed image.
+node: {
+  calculator: "ImageTransformationCalculator"
+  input_stream: "IMAGE:box_image"
+  output_stream: "IMAGE:transformed_box_image"
+  output_stream: "LETTERBOX_PADDING:letterbox_padding"
+  options: {
+    [mediapipe.ImageTransformationCalculatorOptions.ext] {
+      output_width: 224
+      output_height: 224
+      scale_mode: FIT
+    }
+  }
+}
+
+# Converts the transformed input image into an image tensor stored as a
+# TfLiteTensor.
+node {
+  calculator: "TfLiteConverterCalculator"
+  input_stream: "IMAGE:transformed_box_image"
+  output_stream: "TENSORS:image_tensor"
+  options: {
+    [mediapipe.TfLiteConverterCalculatorOptions.ext] {
+      zero_center: false
+    }
+  }
+}
+
+# Generates a single side packet containing a TensorFlow Lite op resolver that
+# supports custom ops needed by the model used in this graph.
+node {
+  calculator: "TfLiteCustomOpResolverCalculator"
+  output_side_packet: "opresolver"
+}
+
+# Runs a TensorFlow Lite model on CPU that takes an image tensor and outputs a
+# vector of tensors representing, for instance, detection boxes/keypoints and
+# scores.
+node {
+  calculator: "TfLiteInferenceCalculator"
+  input_stream: "TENSORS:image_tensor"
+  output_stream: "TENSORS:output_tensors"
+  input_side_packet: "CUSTOM_OP_RESOLVER:opresolver"
+  input_side_packet: "MODEL:model"
+  options: {
+    [mediapipe.TfLiteInferenceCalculatorOptions.ext] {
+      use_gpu: false
+    }
+  }
+}
+
+# Splits a vector of tensors into multiple vectors.
+node {
+  calculator: "SplitTfLiteTensorVectorCalculator"
+  input_stream: "output_tensors"
+  output_stream: "landmark_tensors"
+  output_stream: "box_flag_tensor"
+  options: {
+    [mediapipe.SplitVectorCalculatorOptions.ext] {
+      ranges: { begin: 0 end: 1 }
+      ranges: { begin: 1 end: 2 }
+    }
+  }
+}
+
+# Converts the box-flag tensor into a float that represents the confidence
+# score of box presence.
+node {
+  calculator: "TfLiteTensorsToFloatsCalculator"
+  input_stream: "TENSORS:box_flag_tensor"
+  output_stream: "FLOAT:box_presence_score"
+}
+
+# Applies a threshold to the confidence score to determine whether a box is
+# present.
+node {
+  calculator: "ThresholdingCalculator"
+  input_stream: "FLOAT:box_presence_score"
+  output_stream: "FLAG:box_presence"
+  options: {
+    [mediapipe.ThresholdingCalculatorOptions.ext] {
+      threshold: 0.1
+    }
+  }
+}
+
+# Decodes the landmark tensors into a list of landmarks, where the landmark
+# coordinates are normalized by the size of the input image to the model.
+node {
+  calculator: "TfLiteTensorsToLandmarksCalculator"
+  input_stream: "TENSORS:landmark_tensors"
+  output_stream: "NORM_LANDMARKS:landmarks"
+  options: {
+    [mediapipe.TfLiteTensorsToLandmarksCalculatorOptions.ext] {
+      num_landmarks: 9
+      input_image_width: 224
+      input_image_height: 224
+    }
+  }
+}
+
+# Adjusts landmarks (already normalized to [0.f, 1.f]) on the letterboxed box
+# image (after image transformation with the FIT scale mode) to the
+# corresponding locations on the same image with the letterbox removed (box
+# image before image transformation).
+node {
+  calculator: "LandmarkLetterboxRemovalCalculator"
+  input_stream: "LANDMARKS:landmarks"
+  input_stream: "LETTERBOX_PADDING:letterbox_padding"
+  output_stream: "LANDMARKS:scaled_landmarks"
+}
+
+# Projects the landmarks from the cropped box image to the corresponding
+# locations on the full image before cropping (input to the graph).
+node {
+  calculator: "LandmarkProjectionCalculator"
+  input_stream: "NORM_LANDMARKS:scaled_landmarks"
+  input_stream: "NORM_RECT:box_rect"
+  output_stream: "NORM_LANDMARKS:box_landmarks"
+}
+
+# Extracts image size from the input images.
+node {
+  calculator: "ImagePropertiesCalculator"
+  input_stream: "IMAGE:input_video"
+  output_stream: "SIZE:image_size"
+}
+
+# Smooth predicted landmarks coordinates.
+node {
+  calculator: "LandmarksSmoothingCalculator"
+  input_stream: "NORM_LANDMARKS:box_landmarks"
+  input_stream: "IMAGE_SIZE:image_size"
+  output_stream: "NORM_FILTERED_LANDMARKS:box_landmarks_filtered"
+  options: {
+    [mediapipe.LandmarksSmoothingCalculatorOptions.ext] {
+      velocity_filter: {
+        window_size: 10
+        velocity_scale: 7.5
+      }
+    }
+  }
+}
+
+# Convert box landmarks to frame annotation.
+node {
+  calculator: "LandmarksToFrameAnnotationCalculator"
+  input_stream: "LANDMARKS:box_landmarks_filtered"
+  output_stream: "FRAME_ANNOTATION:box_annotation"
+}
+
+# Lift the 2D landmarks to 3D using EPnP algorithm.
+node {
+  calculator: "Lift2DFrameAnnotationTo3DCalculator"
+  input_stream: "FRAME_ANNOTATION:box_annotation"
+  output_stream: "LIFTED_FRAME_ANNOTATION:lifted_box"
+}
+
+# Get rotated rectangle from lifted box.
+node {
+  calculator: "FrameAnnotationToRectCalculator"
+  input_stream: "FRAME_ANNOTATION:lifted_box"
+  output_stream: "NORM_RECT:rect_from_box"
+}
+
+# Expands the box rectangle so that in the next video frame it's likely to
+# still contain the box even with some motion.
+node {
+  calculator: "RectTransformationCalculator"
+  input_stream: "NORM_RECT:rect_from_box"
+  input_stream: "IMAGE_SIZE:image_size"
+  output_stream: "box_rect_for_next_frame"
+  options: {
+    [mediapipe.RectTransformationCalculatorOptions.ext] {
+      scale_x: 1.5
+      scale_y: 1.5
+      square_long: true
+    }
+  }
+}

mediapipe/modules/objectron/box_landmark_gpu.pbtxt

@@ -0,0 +1,205 @@
+# MediaPipe Box landmark localization GPU subgraph.
+
+type: "BoxLandmarkSubgraph"
+
+input_stream: "IMAGE:input_video"
+input_stream: "NORM_RECT:box_rect"
+output_stream: "FRAME_ANNOTATION:lifted_box"
+output_stream: "NORM_RECT:box_rect_for_next_frame"
+output_stream: "PRESENCE:box_presence"
+
+# Crops the rectangle that contains a box from the input image.
+node {
+  calculator: "ImageCroppingCalculator"
+  input_stream: "IMAGE_GPU:input_video"
+  input_stream: "NORM_RECT:box_rect"
+  output_stream: "IMAGE_GPU:box_image"
+  options: {
+    [mediapipe.ImageCroppingCalculatorOptions.ext] {
+      border_mode: BORDER_REPLICATE
+    }
+  }
+}
+
+# Transforms the input image on GPU to a 224x224 image. To scale the input
+# image, the scale_mode option is set to FIT to preserve the aspect ratio,
+# resulting in potential letterboxing in the transformed image.
+node: {
+  calculator: "ImageTransformationCalculator"
+  input_stream: "IMAGE_GPU:box_image"
+  output_stream: "IMAGE_GPU:transformed_box_image"
+  output_stream: "LETTERBOX_PADDING:letterbox_padding"
+  options: {
+    [mediapipe.ImageTransformationCalculatorOptions.ext] {
+      output_width: 224
+      output_height: 224
+      scale_mode: FIT
+    }
+  }
+}
+
+# Converts the transformed input image on GPU into an image tensor stored as a
+# TfLiteTensor.
+node {
+  calculator: "TfLiteConverterCalculator"
+  input_stream: "IMAGE_GPU:transformed_box_image"
+  output_stream: "TENSORS_GPU:image_tensor"
+  options: {
+    [mediapipe.TfLiteConverterCalculatorOptions.ext] {
+      zero_center: false
+    }
+  }
+}
+
+# Generates a single side packet containing a TensorFlow Lite op resolver that
+# supports custom ops needed by the model used in this graph.
+node {
+  calculator: "TfLiteCustomOpResolverCalculator"
+  output_side_packet: "opresolver"
+}
+
+# Runs a TensorFlow Lite model on GPU that takes an image tensor and outputs a
+# vector of tensors representing, for instance, detection boxes/keypoints and
+# scores.
+node {
+  calculator: "TfLiteInferenceCalculator"
+  input_stream: "TENSORS_GPU:image_tensor"
+  output_stream: "TENSORS:output_tensors"
+  input_side_packet: "CUSTOM_OP_RESOLVER:opresolver"
+  options: {
+    [mediapipe.TfLiteInferenceCalculatorOptions.ext] {
+      model_path: "object_detection_3d.tflite"
+      use_gpu: true
+    }
+  }
+}
+
+# Splits a vector of tensors into multiple vectors.
+node {
+  calculator: "SplitTfLiteTensorVectorCalculator"
+  input_stream: "output_tensors"
+  output_stream: "landmark_tensors"
+  output_stream: "box_flag_tensor"
+  options: {
+    [mediapipe.SplitVectorCalculatorOptions.ext] {
+      ranges: { begin: 0 end: 1 }
+      ranges: { begin: 1 end: 2 }
+    }
+  }
+}
+
+# Converts the box-flag tensor into a float that represents the confidence
+# score of box presence.
+node {
+  calculator: "TfLiteTensorsToFloatsCalculator"
+  input_stream: "TENSORS:box_flag_tensor"
+  output_stream: "FLOAT:box_presence_score"
+}
+
+# Applies a threshold to the confidence score to determine whether a box is
+# present.
+node {
+  calculator: "ThresholdingCalculator"
+  input_stream: "FLOAT:box_presence_score"
+  output_stream: "FLAG:box_presence"
+  options: {
+    [mediapipe.ThresholdingCalculatorOptions.ext] {
+      threshold: 0.99
+    }
+  }
+}
+
+# Decodes the landmark tensors into a list of landmarks, where the landmark
+# coordinates are normalized by the size of the input image to the model.
+node {
+  calculator: "TfLiteTensorsToLandmarksCalculator"
+  input_stream: "TENSORS:landmark_tensors"
+  output_stream: "NORM_LANDMARKS:landmarks"
+  options: {
+    [mediapipe.TfLiteTensorsToLandmarksCalculatorOptions.ext] {
+      num_landmarks: 9
+      input_image_width: 224
+      input_image_height: 224
+    }
+  }
+}
+
+# Adjusts landmarks (already normalized to [0.f, 1.f]) on the letterboxed box
+# image (after image transformation with the FIT scale mode) to the
+# corresponding locations on the same image with the letterbox removed (box
+# image before image transformation).
+node {
+  calculator: "LandmarkLetterboxRemovalCalculator"
+  input_stream: "LANDMARKS:landmarks"
+  input_stream: "LETTERBOX_PADDING:letterbox_padding"
+  output_stream: "LANDMARKS:scaled_landmarks"
+}
+
+# Projects the landmarks from the cropped box image to the corresponding
+# locations on the full image before cropping (input to the graph).
+node {
+  calculator: "LandmarkProjectionCalculator"
+  input_stream: "NORM_LANDMARKS:scaled_landmarks"
+  input_stream: "NORM_RECT:box_rect"
+  output_stream: "NORM_LANDMARKS:box_landmarks"
+}
+
+# Extracts image size from the input images.
+node {
+  calculator: "ImagePropertiesCalculator"
+  input_stream: "IMAGE_GPU:input_video"
+  output_stream: "SIZE:image_size"
+}
+
+# Smooth predicted landmarks coordinates.
+node {
+  calculator: "LandmarksSmoothingCalculator"
+  input_stream: "NORM_LANDMARKS:box_landmarks"
+  input_stream: "IMAGE_SIZE:image_size"
+  output_stream: "NORM_FILTERED_LANDMARKS:box_landmarks_filtered"
+  options: {
+    [mediapipe.LandmarksSmoothingCalculatorOptions.ext] {
+      velocity_filter: {
+        window_size: 10
+        velocity_scale: 7.5
+      }
+    }
+  }
+}
+
+# Convert box landmarks to frame annotation.
+node {
+  calculator: "LandmarksToFrameAnnotationCalculator"
+  input_stream: "LANDMARKS:box_landmarks_filtered"
+  output_stream: "FRAME_ANNOTATION:box_annotation"
+}
+
+# Lift the 2D landmarks to 3D using EPnP algorithm.
+node {
+  calculator: "Lift2DFrameAnnotationTo3DCalculator"
+  input_stream: "FRAME_ANNOTATION:box_annotation"
+  output_stream: "LIFTED_FRAME_ANNOTATION:lifted_box"
+}
+
+# Get rotated rectangle from lifted box.
+node {
+  calculator: "FrameAnnotationToRectCalculator"
+  input_stream: "FRAME_ANNOTATION:lifted_box"
+  output_stream: "NORM_RECT:rect_from_box"
+}
+
+# Expands the box rectangle so that in the next video frame it's likely to
+# still contain the box even with some motion.
+node {
+  calculator: "RectTransformationCalculator"
+  input_stream: "NORM_RECT:rect_from_box"
+  input_stream: "IMAGE_SIZE:image_size"
+  output_stream: "box_rect_for_next_frame"
+  options: {
+    [mediapipe.RectTransformationCalculatorOptions.ext] {
+      scale_x: 1.5
+      scale_y: 1.5
+      square_long: true
+    }
+  }
+}

mediapipe/modules/objectron/calculators/BUILD

@@ -0,0 +1,380 @@
+# Copyright 2020 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library")
+
+licenses(["notice"])
+
+package(default_visibility = ["//visibility:public"])
+
+mediapipe_proto_library(
+    name = "object_proto",
+    srcs = ["object.proto"],
+    visibility = ["//visibility:public"],
+)
+
+mediapipe_proto_library(
+    name = "a_r_capture_metadata_proto",
+    srcs = ["a_r_capture_metadata.proto"],
+    visibility = ["//visibility:public"],
+)
+
+mediapipe_proto_library(
+    name = "annotation_proto",
+    srcs = ["annotation_data.proto"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":a_r_capture_metadata_proto",
+        ":object_proto",
+    ],
+)
+
+mediapipe_proto_library(
+    name = "camera_parameters_proto",
+    srcs = ["camera_parameters.proto"],
+    visibility = ["//visibility:public"],
+)
+
+mediapipe_proto_library(
+    name = "frame_annotation_tracker_calculator_proto",
+    srcs = ["frame_annotation_tracker_calculator.proto"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_options_proto",
+        "//mediapipe/framework:calculator_proto",
+    ],
+)
+
+mediapipe_proto_library(
+    name = "belief_decoder_config_proto",
+    srcs = ["belief_decoder_config.proto"],
+    visibility = ["//visibility:public"],
+)
+
+mediapipe_proto_library(
+    name = "tflite_tensors_to_objects_calculator_proto",
+    srcs = ["tflite_tensors_to_objects_calculator.proto"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":belief_decoder_config_proto",
+        "//mediapipe/framework:calculator_options_proto",
+        "//mediapipe/framework:calculator_proto",
+    ],
+)
+
+mediapipe_proto_library(
+    name = "tensors_to_objects_calculator_proto",
+    srcs = ["tensors_to_objects_calculator.proto"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":belief_decoder_config_proto",
+        "//mediapipe/framework:calculator_options_proto",
+        "//mediapipe/framework:calculator_proto",
+    ],
+)
+
+mediapipe_proto_library(
+    name = "lift_2d_frame_annotation_to_3d_calculator_proto",
+    srcs = ["lift_2d_frame_annotation_to_3d_calculator.proto"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":belief_decoder_config_proto",
+        "//mediapipe/framework:calculator_options_proto",
+        "//mediapipe/framework:calculator_proto",
+    ],
+)
+
+mediapipe_proto_library(
+    name = "frame_annotation_to_rect_calculator_proto",
+    srcs = ["frame_annotation_to_rect_calculator.proto"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_options_proto",
+        "//mediapipe/framework:calculator_proto",
+    ],
+)
+
+mediapipe_proto_library(
+    name = "filter_detection_calculator_proto",
+    srcs = ["filter_detection_calculator.proto"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_options_proto",
+        "//mediapipe/framework:calculator_proto",
+    ],
+)
+
+cc_library(
+    name = "box_util",
+    srcs = ["box_util.cc"],
+    hdrs = ["box_util.h"],
+    deps = [
+        "//mediapipe/framework/port:logging",
+        "//mediapipe/framework/port:opencv_core",
+        "//mediapipe/framework/port:opencv_imgproc",
+        "//mediapipe/util/tracking:box_tracker_cc_proto",
+    ],
+)
+
+cc_library(
+    name = "frame_annotation_tracker",
+    srcs = ["frame_annotation_tracker.cc"],
+    hdrs = ["frame_annotation_tracker.h"],
+    deps = [
+        ":annotation_cc_proto",
+        ":box_util",
+        "//mediapipe/framework/port:integral_types",
+        "//mediapipe/framework/port:logging",
+        "//mediapipe/util/tracking:box_tracker_cc_proto",
+        "@com_google_absl//absl/container:btree",
+        "@com_google_absl//absl/container:flat_hash_set",
+    ],
+)
+
+cc_library(
+    name = "decoder",
+    srcs = [
+        "decoder.cc",
+    ],
+    hdrs = [
+        "decoder.h",
+    ],
+    deps = [
+        ":annotation_cc_proto",
+        ":belief_decoder_config_cc_proto",
+        "//mediapipe/framework/port:logging",
+        "//mediapipe/framework/port:opencv_core",
+        "//mediapipe/framework/port:opencv_imgproc",
+        "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/status",
+        "@eigen_archive//:eigen",
+    ],
+)
+
+cc_library(
+    name = "tensor_util",
+    srcs = [
+        "tensor_util.cc",
+    ],
+    hdrs = [
+        "tensor_util.h",
+    ],
+    deps = [
+        "//mediapipe/framework/formats:tensor",
+        "//mediapipe/framework/port:logging",
+        "//mediapipe/framework/port:opencv_core",
+        "@org_tensorflow//tensorflow/lite:framework",
+    ],
+)
+
+cc_library(
+    name = "box",
+    srcs = [
+        "box.cc",
+        "model.cc",
+    ],
+    hdrs = [
+        "box.h",
+        "model.h",
+        "types.h",
+    ],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":annotation_cc_proto",
+        ":object_cc_proto",
+        "//mediapipe/framework/port:logging",
+        "@eigen_archive//:eigen",
+    ],
+)
+
+cc_library(
+    name = "frame_annotation_to_timed_box_list_calculator",
+    srcs = ["frame_annotation_to_timed_box_list_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":annotation_cc_proto",
+        ":box_util",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/port:opencv_core",
+        "//mediapipe/framework/port:opencv_imgproc",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/util/tracking:box_tracker_cc_proto",
+        "@com_google_absl//absl/memory",
+    ],
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "frame_annotation_tracker_calculator",
+    srcs = ["frame_annotation_tracker_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":annotation_cc_proto",
+        ":frame_annotation_tracker",
+        ":frame_annotation_tracker_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/util/tracking:box_tracker_cc_proto",
+        "@com_google_absl//absl/container:flat_hash_set",
+        "@com_google_absl//absl/memory",
+    ],
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "tflite_tensors_to_objects_calculator",
+    srcs = ["tflite_tensors_to_objects_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":annotation_cc_proto",
+        ":belief_decoder_config_cc_proto",
+        ":decoder",
+        ":tensor_util",
+        ":tflite_tensors_to_objects_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/deps:file_path",
+        "//mediapipe/framework/formats:detection_cc_proto",
+        "//mediapipe/framework/port:opencv_core",
+        "//mediapipe/framework/port:ret_check",
+        "@com_google_absl//absl/memory",
+        "@com_google_absl//absl/strings:str_format",
+        "@com_google_absl//absl/types:span",
+        "@eigen_archive//:eigen",
+        "@org_tensorflow//tensorflow/lite:framework",
+    ],
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "tensors_to_objects_calculator",
+    srcs = ["tensors_to_objects_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":annotation_cc_proto",
+        ":belief_decoder_config_cc_proto",
+        ":decoder",
+        ":tensor_util",
+        ":tensors_to_objects_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/deps:file_path",
+        "//mediapipe/framework/formats:detection_cc_proto",
+        "//mediapipe/framework/port:opencv_core",
+        "//mediapipe/framework/port:ret_check",
+        "@com_google_absl//absl/memory",
+        "@com_google_absl//absl/strings:str_format",
+        "@com_google_absl//absl/types:span",
+        "@eigen_archive//:eigen",
+    ],
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "lift_2d_frame_annotation_to_3d_calculator",
+    srcs = ["lift_2d_frame_annotation_to_3d_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":annotation_cc_proto",
+        ":belief_decoder_config_cc_proto",
+        ":decoder",
+        ":lift_2d_frame_annotation_to_3d_calculator_cc_proto",
+        ":tensor_util",
+        ":tflite_tensors_to_objects_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/deps:file_path",
+        "//mediapipe/framework/formats:detection_cc_proto",
+        "//mediapipe/framework/port:opencv_core",
+        "//mediapipe/framework/port:ret_check",
+        "@com_google_absl//absl/memory",
+        "@com_google_absl//absl/strings:str_format",
+        "@com_google_absl//absl/types:span",
+        "@eigen_archive//:eigen",
+        "@org_tensorflow//tensorflow/lite:framework",
+    ],
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "frame_annotation_to_rect_calculator",
+    srcs = ["frame_annotation_to_rect_calculator.cc"],
+    deps = [
+        ":annotation_cc_proto",
+        ":box",
+        ":frame_annotation_to_rect_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:rect_cc_proto",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/memory",
+        "@eigen_archive//:eigen",
+    ],
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "landmarks_to_frame_annotation_calculator",
+    srcs = ["landmarks_to_frame_annotation_calculator.cc"],
+    deps = [
+        ":annotation_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:landmark_cc_proto",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/memory",
+    ],
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "filter_detection_calculator",
+    srcs = ["filter_detection_calculator.cc"],
+    deps = [
+        ":filter_detection_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:detection_cc_proto",
+        "//mediapipe/framework/formats:location_data_cc_proto",
+        "//mediapipe/framework/port:logging",
+        "//mediapipe/framework/port:map_util",
+        "//mediapipe/framework/port:re2",
+        "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/container:node_hash_set",
+        "@com_google_absl//absl/strings",
+    ],
+    alwayslink = 1,
+)
+
+cc_test(
+    name = "box_util_test",
+    srcs = ["box_util_test.cc"],
+    deps = [
+        ":box_util",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:opencv_core",
+        "//mediapipe/util/tracking:box_tracker_cc_proto",
+    ],
+)
+
+cc_test(
+    name = "frame_annotation_tracker_test",
+    srcs = ["frame_annotation_tracker_test.cc"],
+    deps = [
+        ":annotation_cc_proto",
+        ":frame_annotation_tracker",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:logging",
+        "//mediapipe/util/tracking:box_tracker_cc_proto",
+        "@com_google_absl//absl/container:flat_hash_set",
+    ],
+)

mediapipe/modules/objectron/calculators/a_r_capture_metadata.proto

@@ -0,0 +1,551 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+// Info about the camera characteristics used to capture images and depth data.
+// See developer.apple.com/documentation/avfoundation/avcameracalibrationdata
+// for more information.
+message AVCameraCalibrationData {
+  // 3x3 row-major matrix relating a camera's internal properties to an ideal
+  // pinhole-camera model.
+  // See
+  // developer.apple.com/documentation/avfoundation/avcameracalibrationdata/2881135-intrinsicmatrix
+  // for detailed usage information.
+  repeated float intrinsic_matrix = 1 [packed = true];
+
+  // The image dimensions to which the intrinsic_matrix values are relative.
+  optional float intrinsic_matrix_reference_dimension_width = 2;
+  optional float intrinsic_matrix_reference_dimension_height = 3;
+
+  // 3x4 row-major matrix relating a camera's position and orientation to a
+  // world or scene coordinate system. Consists of a unitless 3x3 rotation
+  // matrix (R) on the left and a translation (t) 3x1 vector on the right. The
+  // translation vector's units are millimeters. For example:
+  //
+  //            |r1,1  r2,1  r3,1 | t1|
+  //  [R | t] = |r1,2  r2,2  r3,2 | t2|
+  //            |r1,3  r2,3  r3,3 | t3|
+  //
+  //  is stored as [r11, r21, r31, t1, r12, r22, r32, t2, ...]
+  //
+  // See
+  // developer.apple.com/documentation/avfoundation/avcameracalibrationdata/2881130-extrinsicmatrix?language=objc
+  // for more information.
+  repeated float extrinsic_matrix = 4 [packed = true];
+
+  // The size, in millimeters, of one image pixel.
+  optional float pixel_size = 5;
+
+  // A list of floating-point values describing radial distortions imparted by
+  // the camera lens, for use in rectifying camera images.
+  // See
+  // developer.apple.com/documentation/avfoundation/avcameracalibrationdata/2881129-lensdistortionlookuptable?language=objc
+  // for more information.
+  repeated float lens_distortion_lookup_values = 6 [packed = true];
+
+  // A list of floating-point values describing radial distortions for use in
+  // reapplying camera geometry to a rectified image.
+  // See
+  // developer.apple.com/documentation/avfoundation/avcameracalibrationdata/2881132-inverselensdistortionlookuptable?language=objc
+  // for more information.
+  repeated float inverse_lens_distortion_lookup_values = 7 [packed = true];
+
+  // The offset of the distortion center of the camera lens from the top-left
+  // corner of the image.
+  // See
+  // developer.apple.com/documentation/avfoundation/avcameracalibrationdata/2881131-lensdistortioncenter?language=objc
+  // for more information.
+  optional float lens_distortion_center_x = 8;
+  optional float lens_distortion_center_y = 9;
+}
+
+// Container for depth data information.
+// See developer.apple.com/documentation/avfoundation/avdepthdata for more info.
+message AVDepthData {
+  // PNG representation of the grayscale depth data map. See discussion about
+  // depth_data_map_original_minimum_value, below, for information about how
+  // to interpret the pixel values.
+  optional bytes depth_data_map = 1;
+
+  // Pixel format type of the original captured depth data.
+  // See
+  // developer.apple.com/documentation/corevideo/1563591-pixel_format_identifiers?language=objc
+  // for the complete list of possible pixel format types. This value represents
+  // a string for the associated OSType/FourCharCode.
+  optional string depth_data_type = 2;
+
+  // Indicates the general accuracy of the depth_data_map.
+  // See developer.apple.com/documentation/avfoundation/avdepthdataaccuracy for
+  // more information.
+  enum Accuracy {
+    UNDEFINED_ACCURACY = 0;
+    // Values in the depth map are usable for foreground/background separation
+    // but are not absolutely accurate in the physical world.
+    RELATIVE = 1;
+    // Values in the depth map are absolutely accurate in the physical world.
+    ABSOLUTE = 2;
+  }
+  optional Accuracy depth_data_accuracy = 3 [default = RELATIVE];
+
+  // Indicates whether the depth_data_map contains temporally smoothed data.
+  optional bool depth_data_filtered = 4;
+
+  // Quality of the depth_data_map.
+  enum Quality {
+    UNDEFINED_QUALITY = 0;
+    HIGH = 1;
+    LOW = 2;
+  }
+  optional Quality depth_data_quality = 5;
+
+  // Associated calibration data for the depth_data_map.
+  optional AVCameraCalibrationData camera_calibration_data = 6;
+
+  // The original range of values expressed by the depth_data_map, before
+  // grayscale normalization. For example, if the minimum and maximum values
+  // indicate a range of [0.5, 2.2], and the depth_data_type value indicates
+  // it was a depth map, then white pixels (255, 255, 255) will map to 0.5 and
+  // black pixels (0, 0, 0) will map to 2.2 with the grayscale range linearly
+  // interpolated inbetween. Conversely, if the depth_data_type value indicates
+  // it was a disparity map, then white pixels will map to 2.2 and black pixels
+  // will map to 0.5.
+  optional float depth_data_map_original_minimum_value = 7;
+  optional float depth_data_map_original_maximum_value = 8;
+
+  // The width of the depth buffer map.
+  optional int32 depth_data_map_width = 9;
+
+  // The height of the depth buffer map.
+  optional int32 depth_data_map_height = 10;
+
+  // The row-major flattened array of the depth buffer map pixels. This will be
+  // either a float32 or float16 byte array, depending on 'depth_data_type'.
+  optional bytes depth_data_map_raw_values = 11;
+}
+
+// Estimated scene lighting information associated with a captured video frame.
+// See developer.apple.com/documentation/arkit/arlightestimate for more info.
+message ARLightEstimate {
+  // The estimated intensity, in lumens, of ambient light throughout the scene.
+  optional double ambient_intensity = 1;
+
+  // The estimated color temperature, in degrees Kelvin, of ambient light
+  // throughout the scene.
+  optional double ambient_color_temperature = 2;
+
+  // Data describing the estimated lighting environment in all directions.
+  // Second-level spherical harmonics in separate red, green, and blue data
+  // planes. Thus, this buffer contains 3 sets of 9 coefficients, or a total of
+  // 27 values.
+  // See
+  // https://developer.apple.com/documentation/arkit/ardirectionallightestimate/2928222-sphericalharmonicscoefficients?language=objc
+  // for more information.
+  repeated float spherical_harmonics_coefficients = 3 [packed = true];
+
+  message DirectionVector {
+    optional float x = 1;
+    optional float y = 2;
+    optional float z = 3;
+  }
+  // A vector indicating the orientation of the strongest directional light
+  // source, normalized in the world-coordinate space.
+  // See
+  // https://developer.apple.com/documentation/arkit/ardirectionallightestimate/2928221-primarylightdirection?language=objc
+  // for more information;
+  optional DirectionVector primary_light_direction = 4;
+
+  // The estimated intensity, in lumens, of the strongest directional light
+  // source in the scene.
+  // See
+  // https://developer.apple.com/documentation/arkit/ardirectionallightestimate/2928219-primarylightintensity?language=objc
+  // for more information.
+  optional float primary_light_intensity = 5;
+}
+
+// Information about the camera position and imaging characteristics for a
+// captured video frame.
+// See developer.apple.com/documentation/arkit/arcamera for more information.
+message ARCamera {
+  // The general quality of position tracking available when the camera captured
+  // a frame.
+  enum TrackingState {
+    UNDEFINED_TRACKING_STATE = 0;
+    // Camera position tracking is not available.
+    UNAVAILABLE = 1;
+    // Tracking is available, but the quality of results is questionable.
+    LIMITED = 2;
+    // Camera position tracking is providing optimal results.
+    NORMAL = 3;
+  }
+  optional TrackingState tracking_state = 1 [default = UNAVAILABLE];
+
+  // A possible diagnosis for limited position tracking quality as of when the
+  // frame was captured.
+  enum TrackingStateReason {
+    UNDEFINED_TRACKING_STATE_REASON = 0;
+    // The current tracking state is not limited.
+    NONE = 1;
+    // Not yet enough camera or motion data to provide tracking information.
+    INITIALIZING = 2;
+    // The device is moving too fast for accurate image-based position tracking.
+    EXCESSIVE_MOTION = 3;
+    // Not enough distinguishable features for image-based position tracking.
+    INSUFFICIENT_FEATURES = 4;
+    // Tracking is limited due to a relocalization in progress.
+    RELOCALIZING = 5;
+  }
+  optional TrackingStateReason tracking_state_reason = 2 [default = NONE];
+
+  // 4x4 row-major matrix expressing position and orientation of the camera in
+  // world coordinate space.
+  // See developer.apple.com/documentation/arkit/arcamera/2866108-transform for
+  // more information.
+  repeated float transform = 3 [packed = true];
+
+  // The orientation of the camera, expressed as roll, pitch, and yaw values.
+  message EulerAngles {
+    optional float roll = 1;
+    optional float pitch = 2;
+    optional float yaw = 3;
+  }
+  optional EulerAngles euler_angles = 4;
+
+  // The width and height, in pixels, of the captured camera image.
+  optional int32 image_resolution_width = 5;
+  optional int32 image_resolution_height = 6;
+
+  // 3x3 row-major matrix that converts between the 2D camera plane and 3D world
+  // coordinate space.
+  // See developer.apple.com/documentation/arkit/arcamera/2875730-intrinsics for
+  // usage information.
+  repeated float intrinsics = 7 [packed = true];
+
+  // 4x4 row-major transform matrix appropriate for rendering 3D content to
+  // match the image captured by the camera.
+  // See
+  // developer.apple.com/documentation/arkit/arcamera/2887458-projectionmatrix
+  // for usage information.
+  repeated float projection_matrix = 8 [packed = true];
+
+  // 4x4 row-major transform matrix appropriate for converting from world-space
+  // to camera space. Relativized for the captured_image orientation (i.e.
+  // UILandscapeOrientationRight).
+  // See
+  // https://developer.apple.com/documentation/arkit/arcamera/2921672-viewmatrixfororientation?language=objc
+  // for more information.
+  repeated float view_matrix = 9 [packed = true];
+}
+
+// Container for a 3D mesh describing face topology.
+message ARFaceGeometry {
+  // Each vertex represents a 3D point in the face mesh, in the face coordinate
+  // space.
+  // See developer.apple.com/documentation/arkit/arfacegeometry/2928201-vertices
+  // for more information.
+  message Vertex {
+    optional float x = 1;
+    optional float y = 2;
+    optional float z = 3;
+  }
+  repeated Vertex vertices = 1;
+
+  // The number of elements in the vertices list.
+  optional int32 vertex_count = 2;
+
+  // Each texture coordinate represents UV texture coordinates for the vertex at
+  // the corresponding index in the vertices buffer.
+  // See
+  // developer.apple.com/documentation/arkit/arfacegeometry/2928203-texturecoordinates
+  // for more information.
+  message TextureCoordinate {
+    optional float u = 1;
+    optional float v = 2;
+  }
+  repeated TextureCoordinate texture_coordinates = 3;
+
+  // The number of elements in the texture_coordinates list.
+  optional int32 texture_coordinate_count = 4;
+
+  // Each integer value in this ordered list represents an index into the
+  // vertices and texture_coordinates lists. Each set of three indices
+  // identifies the vertices comprising a single triangle in the mesh. Each set
+  // of three indices forms a triangle, so the number of indices in the
+  // triangle_indices buffer is three times the triangle_count value.
+  // See
+  // developer.apple.com/documentation/arkit/arfacegeometry/2928199-triangleindices
+  // for more information.
+  repeated int32 triangle_indices = 5 [packed = true];
+
+  // The number of triangles described by the triangle_indices buffer.
+  // See
+  // developer.apple.com/documentation/arkit/arfacegeometry/2928207-trianglecount
+  // for more information.
+  optional int32 triangle_count = 6;
+}
+
+// Contains a list of blend shape entries wherein each item maps a specific
+// blend shape location to its associated coefficient.
+message ARBlendShapeMap {
+  message MapEntry {
+    // Identifier for the specific facial feature.
+    // See developer.apple.com/documentation/arkit/arblendshapelocation for a
+    // complete list of identifiers.
+    optional string blend_shape_location = 1;
+
+    // Indicates the current position of the feature relative to its neutral
+    // configuration, ranging from 0.0 (neutral) to 1.0 (maximum movement).
+    optional float blend_shape_coefficient = 2;
+  }
+  repeated MapEntry entries = 1;
+}
+
+// Information about the pose, topology, and expression of a detected face.
+// See developer.apple.com/documentation/arkit/arfaceanchor for more info.
+message ARFaceAnchor {
+  // A coarse triangle mesh representing the topology of the detected face.
+  optional ARFaceGeometry geometry = 1;
+
+  // A map of named coefficients representing the detected facial expression in
+  // terms of the movement of specific facial features.
+  optional ARBlendShapeMap blend_shapes = 2;
+
+  // 4x4 row-major matrix encoding the position, orientation, and scale of the
+  // anchor relative to the world coordinate space.
+  // See
+  // https://developer.apple.com/documentation/arkit/aranchor/2867981-transform?language=objc
+  // for more information.
+  repeated float transform = 3;
+
+  // Indicates whether the anchor's transform is valid. Frames that have a face
+  // anchor with this value set to NO should probably be ignored.
+  optional bool is_tracked = 4;
+}
+
+// Container for a 3D mesh.
+message ARPlaneGeometry {
+  message Vertex {
+    optional float x = 1;
+    optional float y = 2;
+    optional float z = 3;
+  }
+
+  // Each texture coordinate represents UV texture coordinates for the vertex at
+  // the corresponding index in the vertices buffer.
+  // See
+  // https://developer.apple.com/documentation/arkit/arfacegeometry/2928203-texturecoordinates
+  // for more information.
+  message TextureCoordinate {
+    optional float u = 1;
+    optional float v = 2;
+  }
+
+  // A buffer of vertex positions for each point in the plane mesh.
+  repeated Vertex vertices = 1;
+
+  // The number of elements in the vertices buffer.
+  optional int32 vertex_count = 2;
+
+  // A buffer of texture coordinate values for each point in the plane mesh.
+  repeated TextureCoordinate texture_coordinates = 3;
+
+  // The number of elements in the texture_coordinates buffer.
+  optional int32 texture_coordinate_count = 4;
+
+  // Each integer value in this ordered list represents an index into the
+  // vertices and texture_coordinates lists. Each set of three indices
+  // identifies the vertices comprising a single triangle in the mesh. Each set
+  // of three indices forms a triangle, so the number of indices in the
+  // triangle_indices buffer is three times the triangle_count value.
+  // See
+  // https://developer.apple.com/documentation/arkit/arplanegeometry/2941051-triangleindices
+  // for more information.
+  repeated int32 triangle_indices = 5 [packed = true];
+
+  // Each set of three indices forms a triangle, so the number of indices in the
+  // triangle_indices buffer is three times the triangle_count value.
+  // See
+  // https://developer.apple.com/documentation/arkit/arplanegeometry/2941058-trianglecount
+  // for more information.
+  optional int32 triangle_count = 6;
+
+  // Each value in this buffer represents the position of a vertex along the
+  // boundary polygon of the estimated plane. The owning plane anchor's
+  // transform matrix defines the coordinate system for these points.
+  // See
+  // https://developer.apple.com/documentation/arkit/arplanegeometry/2941052-boundaryvertices
+  // for more information.
+  repeated Vertex boundary_vertices = 7;
+
+  // The number of elements in the boundary_vertices buffer.
+  optional int32 boundary_vertex_count = 8;
+}
+
+// Information about the position and orientation of a real-world flat surface.
+// See https://developer.apple.com/documentation/arkit/arplaneanchor for more
+// information.
+message ARPlaneAnchor {
+  enum Alignment {
+    UNDEFINED = 0;
+    // The plane is perpendicular to gravity.
+    HORIZONTAL = 1;
+    // The plane is parallel to gravity.
+    VERTICAL = 2;
+  }
+
+  // Wrapper for a 3D point / vector within the plane. See extent and center
+  // values for more information.
+  message PlaneVector {
+    optional float x = 1;
+    optional float y = 2;
+    optional float z = 3;
+  }
+
+  enum PlaneClassification {
+    NONE = 0;
+    WALL = 1;
+    FLOOR = 2;
+    CEILING = 3;
+    TABLE = 4;
+    SEAT = 5;
+  }
+
+  // The classification status for the plane.
+  enum PlaneClassificationStatus {
+    // The classfication process for the plane anchor has completed but the
+    // result is inconclusive.
+    UNKNOWN = 0;
+    // No classication information can be provided (set on error or if the
+    // device does not support plane classification).
+    UNAVAILABLE = 1;
+    // The classification process has not completed.
+    UNDETERMINED = 2;
+    // The classfication process for the plane anchor has completed.
+    KNOWN = 3;
+  }
+
+  // The ID of the plane.
+  optional string identifier = 1;
+
+  // 4x4 row-major matrix encoding the position, orientation, and scale of the
+  // anchor relative to the world coordinate space.
+  // See
+  // https://developer.apple.com/documentation/arkit/aranchor/2867981-transform
+  // for more information.
+  repeated float transform = 2;
+
+  // The general orientation of the detected plane with respect to gravity.
+  optional Alignment alignment = 3;
+
+  // A coarse triangle mesh representing the general shape of the detected
+  // plane.
+  optional ARPlaneGeometry geometry = 4;
+
+  // The center point of the plane relative to its anchor position.
+  // Although the type of this property is a 3D vector, a plane anchor is always
+  // two-dimensional, and is always positioned in only the x and z directions
+  // relative to its transform position. (That is, the y-component of this
+  // vector is always zero.)
+  // See
+  // https://developer.apple.com/documentation/arkit/arplaneanchor/2882056-center
+  // for more information.
+  optional PlaneVector center = 5;
+
+  // The estimated width and length of the detected plane.
+  // See
+  // https://developer.apple.com/documentation/arkit/arplaneanchor/2882055-extent
+  // for more information.
+  optional PlaneVector extent = 6;
+
+  // A Boolean value that indicates whether plane classification is available on
+  // the current device. On devices without plane classification support, all
+  // plane anchors report a classification value of NONE
+  // and a classification_status value of UNAVAILABLE.
+  optional bool classification_supported = 7;
+
+  // A general characterization of what kind of real-world surface the plane
+  // anchor represents.
+  // See
+  // https://developer.apple.com/documentation/arkit/arplaneanchor/2990936-classification
+  // for more information.
+  optional PlaneClassification classification = 8;
+
+  // The current state of ARKit's process for classifying the plane anchor.
+  // When this property's value is KNOWN, the classification property represents
+  // ARKit's characterization of the real-world surface corresponding to the
+  // plane anchor.
+  // See
+  // https://developer.apple.com/documentation/arkit/arplaneanchor/2990937-classificationstatus
+  // for more information.
+  optional PlaneClassificationStatus classification_status = 9;
+}
+
+// A collection of points in the world coordinate space.
+// See https://developer.apple.com/documentation/arkit/arpointcloud for more
+// information.
+message ARPointCloud {
+  message Point {
+    optional float x = 1;
+    optional float y = 2;
+    optional float z = 3;
+  }
+
+  // The number of points in the cloud.
+  optional int32 count = 1;
+
+  // The list of detected points.
+  repeated Point point = 2;
+
+  // A list of unique identifiers corresponding to detected feature points.
+  // Each identifier in this list corresponds to the point at the same index
+  // in the points array.
+  repeated int64 identifier = 3 [packed = true];
+}
+
+// Video image and face position tracking information.
+// See developer.apple.com/documentation/arkit/arframe for more information.
+message ARFrame {
+  // The timestamp for the frame.
+  optional double timestamp = 1;
+
+  // The depth data associated with the frame. Not all frames have depth data.
+  optional AVDepthData depth_data = 2;
+
+  // The depth data object timestamp associated with the frame. May differ from
+  // the frame timestamp value. Is only set when the frame has depth_data.
+  optional double depth_data_timestamp = 3;
+
+  // Camera information associated with the frame.
+  optional ARCamera camera = 4;
+
+  // Light information associated with the frame.
+  optional ARLightEstimate light_estimate = 5;
+
+  // Face anchor information associated with the frame. Not all frames have an
+  // active face anchor.
+  optional ARFaceAnchor face_anchor = 6;
+
+  // Plane anchors associated with the frame. Not all frames have a plane
+  // anchor. Plane anchors and face anchors are mutually exclusive.
+  repeated ARPlaneAnchor plane_anchor = 7;
+
+  // The current intermediate results of the scene analysis used to perform
+  // world tracking.
+  // See
+  // https://developer.apple.com/documentation/arkit/arframe/2887449-rawfeaturepoints
+  // for more information.
+  optional ARPointCloud raw_feature_points = 8;
+}

mediapipe/modules/objectron/calculators/annotation_data.proto

@@ -0,0 +1,92 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto3";
+
+package mediapipe;
+
+import "mediapipe/modules/objectron/calculators/a_r_capture_metadata.proto";
+import "mediapipe/modules/objectron/calculators/object.proto";
+
+// Projection of a 3D point on an image, and its metric depth.
+message NormalizedPoint2D {
+  // x-y position of the 2d keypoint in the image coordinate system.
+  // u,v \in [0, 1], where top left corner is (0, 0) and the bottom-right corner
+  // is (1, 1).
+  float x = 1;
+  float y = 2;
+
+  // The depth of the point in the camera coordinate system (in meters).
+  float depth = 3;
+}
+
+// The 3D point in the camera coordinate system, the scales are in meters.
+message Point3D {
+  float x = 1;
+  float y = 2;
+  float z = 3;
+}
+
+message AnnotatedKeyPoint {
+  int32 id = 1;
+  Point3D point_3d = 2;
+  NormalizedPoint2D point_2d = 3;
+}
+
+message ObjectAnnotation {
+  // Reference to the object identifier in ObjectInstance.
+  int32 object_id = 1;
+
+  // For each objects, list all the annotated keypoints here.
+  // E.g. for bounding-boxes, we have 8 keypoints, hands = 21 keypoints, etc.
+  // These normalized points are the projection of the Object's 3D keypoint
+  // on the current frame's camera poses.
+  repeated AnnotatedKeyPoint keypoints = 2;
+
+  // Visibiity of this annotation in a frame.
+  float visibility = 3;
+}
+
+message FrameAnnotation {
+  // Unique frame id, corresponds to images.
+  int32 frame_id = 1;
+
+  // List of the annotated objects in this frame. Depending on how many object
+  // are observable in this frame, we might have non or as much as
+  // sequence.objects_size() annotations.
+  repeated ObjectAnnotation annotations = 2;
+
+  // Information about the camera transformation (in the world coordinate) and
+  // imaging characteristics for a captured video frame.
+  ARCamera camera = 3;
+
+  // The timestamp for the frame.
+  double timestamp = 4;
+
+  // Plane center and normal in camera frame.
+  repeated float plane_center = 5;
+  repeated float plane_normal = 6;
+}
+
+// The sequence protocol contains the annotation data for the entire video clip.
+message Sequence {
+  // List of all the annotated 3D objects in this sequence in the world
+  // Coordinate system. Given the camera poses of each frame (also in the
+  // world-coordinate) these objects bounding boxes can be projected to each
+  // frame to get the per-frame annotation (i.e. image_annotation below).
+  repeated Object objects = 1;
+
+  // List of annotated data per each frame in sequence + frame information.
+  repeated FrameAnnotation frame_annotations = 2;
+}

mediapipe/modules/objectron/calculators/belief_decoder_config.proto

@@ -0,0 +1,38 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+message BeliefDecoderConfig {
+  optional float heatmap_threshold = 1 [default = 0.9];
+  // Maximum distance in pixels between two local max heatmap values.
+  optional float local_max_distance = 2 [default = 10.0];
+  // Coefficient of offset_scale.
+  // offset_scale = offset_scale_coef * min(rows, cols).
+  // offset_scale is used to multiply the offset predictions from the network.
+  optional float offset_scale_coef = 3 [default = 0.5, deprecated = true];
+
+  // The radius for vertex voting. Use no voting if the radius is less than or
+  // euqal to 1. Example: 10.
+  optional int32 voting_radius = 4;
+
+  // The number of pixels to determine whether two points are the same.
+  // Example: 5 (voting_radius / 2).
+  optional int32 voting_allowance = 5;
+
+  // The threshold of beliefs, with which the points can vote. Example: 0.2.
+  optional float voting_threshold = 6;
+}

mediapipe/modules/objectron/calculators/box.cc

@@ -0,0 +1,255 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/modules/objectron/calculators/box.h"
+
+#include "Eigen/src/Core/util/Constants.h"
+#include "mediapipe/framework/port/logging.h"
+
+namespace mediapipe {
+
+namespace {
+constexpr int kFrontFaceId = 4;
+constexpr int kTopFaceId = 2;
+constexpr int kNumKeypoints = 8 + 1;
+constexpr int kNumberOfAxis = 3;
+constexpr int kEdgesPerAxis = 4;
+
+}  // namespace
+
+Box::Box(const std::string& category)
+    : Model(kBoundingBox, kNumKeypoints, category),
+      bounding_box_(kNumKeypoints) {
+  transformation_.setIdentity();
+
+  scale_ << 0.1, 0.1, 0.1;
+
+  // The vertices are ordered according to the left-hand rule, so the normal
+  // vector of each face will point inward the box.
+  faces_.push_back({5, 6, 8, 7});  // +x on yz plane
+  faces_.push_back({1, 3, 4, 2});  // -x on yz plane
+
+  faces_.push_back({3, 7, 8, 4});  // +y on xz plane = top
+  faces_.push_back({1, 2, 6, 5});  // -y on xz plane
+
+  faces_.push_back({2, 4, 8, 6});  // +z on xy plane = front
+  faces_.push_back({1, 5, 7, 3});  // -z on xy plane
+
+  // Add the edges in the cube, they are sorted according to axis (x-y-z).
+  edges_.push_back({1, 5});
+  edges_.push_back({2, 6});
+  edges_.push_back({3, 7});
+  edges_.push_back({4, 8});
+
+  edges_.push_back({1, 3});
+  edges_.push_back({5, 7});
+  edges_.push_back({2, 4});
+  edges_.push_back({6, 8});
+
+  edges_.push_back({1, 2});
+  edges_.push_back({3, 4});
+  edges_.push_back({5, 6});
+  edges_.push_back({7, 8});
+  Update();
+}
+
+void Box::Update() {
+  // Compute the eight vertices of the bounding box from Box's parameters
+  auto w = scale_[0] / 2.f;
+  auto h = scale_[1] / 2.f;
+  auto d = scale_[2] / 2.f;
+
+  // Define the local coordinate system, w.r.t. the center of the boxs
+  bounding_box_[0] << 0., 0., 0.;
+  bounding_box_[1] << -w, -h, -d;
+  bounding_box_[2] << -w, -h, +d;
+  bounding_box_[3] << -w, +h, -d;
+  bounding_box_[4] << -w, +h, +d;
+  bounding_box_[5] << +w, -h, -d;
+  bounding_box_[6] << +w, -h, +d;
+  bounding_box_[7] << +w, +h, -d;
+  bounding_box_[8] << +w, +h, +d;
+
+  // Convert to world coordinate system
+  for (int i = 0; i < kNumKeypoints; ++i) {
+    bounding_box_[i] =
+        transformation_.topLeftCorner<3, 3>() * bounding_box_[i] +
+        transformation_.col(3).head<3>();
+  }
+}
+
+void Box::Adjust(const std::vector<float>& variables) {
+  Eigen::Vector3f translation;
+  translation << variables[0], variables[1], variables[2];
+  SetTranslation(translation);
+
+  const float roll = variables[3];
+  const float pitch = variables[4];
+  const float yaw = variables[5];
+  SetRotation(roll, pitch, yaw);
+
+  Eigen::Vector3f scale;
+  scale << variables[6], variables[7], variables[8];
+
+  SetScale(scale);
+  Update();
+}
+
+float* Box::GetVertex(size_t vertex_id) {
+  CHECK_LT(vertex_id, kNumKeypoints);
+  return bounding_box_[vertex_id].data();
+}
+
+const float* Box::GetVertex(size_t vertex_id) const {
+  CHECK_LT(vertex_id, kNumKeypoints);
+  return bounding_box_[vertex_id].data();
+}
+
+bool Box::InsideTest(const Eigen::Vector3f& point, int check_axis) const {
+  const float* v0 = GetVertex(1);
+  const float* v1 = GetVertex(2);
+  const float* v2 = GetVertex(3);
+  const float* v4 = GetVertex(5);
+
+  switch (check_axis) {
+    case 1:
+      return (v0[0] <= point[0] && point[0] <= v1[0]);  // X-axis
+    case 2:
+      return (v0[1] <= point[1] && point[1] <= v2[1]);  // Y-axis
+    case 3:
+      return (v0[2] <= point[2] && point[2] <= v4[2]);  // Z-axis
+    default:
+      return false;
+  }
+}
+
+void Box::Deserialize(const Object& obj) {
+  CHECK_EQ(obj.keypoints_size(), kNumKeypoints);
+  Model::Deserialize(obj);
+}
+
+void Box::Serialize(Object* obj) {
+  Model::Serialize(obj);
+  obj->set_type(Object::BOUNDING_BOX);
+  std::vector<Vector3f> local_bounding_box(9);
+  // Define the local coordinate system, w.r.t. the center of the boxs
+  local_bounding_box[0] << 0., 0., 0.;
+  local_bounding_box[1] << -0.5, -0.5, -0.5;
+  local_bounding_box[2] << -0.5, -0.5, +0.5;
+  local_bounding_box[3] << -0.5, +0.5, -0.5;
+  local_bounding_box[4] << -0.5, +0.5, +0.5;
+  local_bounding_box[5] << +0.5, -0.5, -0.5;
+  local_bounding_box[6] << +0.5, -0.5, +0.5;
+  local_bounding_box[7] << +0.5, +0.5, -0.5;
+  local_bounding_box[8] << +0.5, +0.5, +0.5;
+  for (int i = 0; i < kNumKeypoints; ++i) {
+    KeyPoint* keypoint = obj->add_keypoints();
+    keypoint->set_x(local_bounding_box[i][0]);
+    keypoint->set_y(local_bounding_box[i][1]);
+    keypoint->set_z(local_bounding_box[i][2]);
+    keypoint->set_confidence_radius(0.);
+  }
+}
+
+const Face& Box::GetFrontFace() const { return faces_[kFrontFaceId]; }
+
+const Face& Box::GetTopFace() const { return faces_[kTopFaceId]; }
+
+std::pair<Vector3f, Vector3f> Box::GetGroundPlane() const {
+  const Vector3f gravity = Vector3f(0., 1., 0.);
+  int ground_plane_id = 0;
+  float ground_plane_error = 10.0;
+
+  auto get_face_center = [&](const Face& face) {
+    Vector3f center = Vector3f::Zero();
+    for (const int vertex_id : face) {
+      center += Map<const Vector3f>(GetVertex(vertex_id));
+    }
+    center /= face.size();
+    return center;
+  };
+
+  auto get_face_normal = [&](const Face& face, const Vector3f& center) {
+    Vector3f v1 = Map<const Vector3f>(GetVertex(face[0])) - center;
+    Vector3f v2 = Map<const Vector3f>(GetVertex(face[1])) - center;
+    Vector3f normal = v1.cross(v2);
+    return normal;
+  };
+
+  // The ground plane is defined as a plane aligned with gravity.
+  // gravity is the (0, 1, 0) vector in the world coordinate system.
+  const auto& faces = GetFaces();
+  for (int face_id = 0; face_id < faces.size(); face_id += 2) {
+    const auto& face = faces[face_id];
+    Vector3f center = get_face_center(face);
+    Vector3f normal = get_face_normal(face, center);
+    Vector3f w = gravity.cross(normal);
+    const float w_sq_norm = w.squaredNorm();
+    if (w_sq_norm < ground_plane_error) {
+      ground_plane_error = w_sq_norm;
+      ground_plane_id = face_id;
+    }
+  }
+
+  Vector3f center = get_face_center(faces[ground_plane_id]);
+  Vector3f normal = get_face_normal(faces[ground_plane_id], center);
+
+  // For each face, we also have a parallel face that it's normal is also
+  // aligned with gravity vector. We pick the face with lower height (y-value).
+  // The parallel to face 0 is 1, face 2 is 3, and face 4 is 5.
+  int parallel_face_id = ground_plane_id + 1;
+  const auto& parallel_face = faces[parallel_face_id];
+  Vector3f parallel_face_center = get_face_center(parallel_face);
+  Vector3f parallel_face_normal =
+      get_face_normal(parallel_face, parallel_face_center);
+  if (parallel_face_center[1] < center[1]) {
+    center = parallel_face_center;
+    normal = parallel_face_normal;
+  }
+  return {center, normal};
+}
+
+template <typename T>
+void Box::Fit(const std::vector<T>& vertices) {
+  CHECK_EQ(vertices.size(), kNumKeypoints);
+  scale_.setZero();
+  // The scale would remain invariant under rotation and translation.
+  // We can safely estimate the scale from the oriented box.
+  for (int axis = 0; axis < kNumberOfAxis; ++axis) {
+    for (int edge_id = 0; edge_id < kEdgesPerAxis; ++edge_id) {
+      // The edges are stored in quadruples according to each axis
+      const std::array<int, 2>& edge = edges_[axis * kEdgesPerAxis + edge_id];
+      scale_[axis] += (vertices[edge[0]] - vertices[edge[1]]).norm();
+    }
+    scale_[axis] /= kEdgesPerAxis;
+  }
+  // Create a scaled axis-aligned box
+  transformation_.setIdentity();
+  Update();
+
+  using MatrixN3_RM = Eigen::Matrix<float, kNumKeypoints, 3, Eigen::RowMajor>;
+  Eigen::Map<const MatrixN3_RM> v(vertices[0].data());
+  Eigen::Map<const MatrixN3_RM> system(bounding_box_[0].data());
+  auto system_h = system.rowwise().homogeneous().eval();
+  auto system_g = system_h.colPivHouseholderQr();
+  auto solution = system_g.solve(v).eval();
+  transformation_.topLeftCorner<3, 4>() = solution.transpose();
+  Update();
+}
+
+template void Box::Fit<Vector3f>(const std::vector<Vector3f>&);
+template void Box::Fit<Map<Vector3f>>(const std::vector<Map<Vector3f>>&);
+template void Box::Fit<Map<const Vector3f>>(
+    const std::vector<Map<const Vector3f>>&);
+}  // namespace mediapipe

mediapipe/modules/objectron/calculators/box.h

@@ -0,0 +1,132 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_MODULES_OBJECTRON_CALCULATORS_BOX_H_
+#define MEDIAPIPE_MODULES_OBJECTRON_CALCULATORS_BOX_H_
+
+#include <vector>
+
+#include "mediapipe/modules/objectron/calculators/model.h"
+
+namespace mediapipe {
+
+// Model for the bounding box in 3D
+// The box has 9 degrees of freedom, which uniquely defines 8 keypoints in the
+// fixed world-coordinate system.
+//
+// The 8 keypoints are defined as follows
+//
+//  kp-id  axis
+//  0      000    ---
+//  1      001    --+
+//  2      010    -+-
+//  3      011    -++
+//  4      100    +--
+//  5      101    +-+
+//  6      110    ++-
+//  7      111    +++
+//
+// where xyz means positive or negative vector along the axis where the center
+// of the box is the origin. The resulting bounding box is
+//
+//              x                              x
+//      0 + + + + + + + + 4                 .-------
+//      +\                +\                |\
+//      + \ y             + \             z | \ y
+//      +  \              +  \              |  \
+//      +   2 + + + + + + + + 6
+//    z +   +             +   +
+//      +   +             +   +
+//      +   +     C       +   +
+//      +   +             +   +
+//      1 + + + + + + + + 5   +
+//       \  +              \  +
+//        \ +               \ +
+//         \+                \+
+//          3 + + + + + + + + 7
+//
+// World coordinate system: +y is up (aligned with gravity),
+// +z is toward the user, +x follows right hand rule.
+// The front face is defined as +z axis on xy plane.
+// The top face is defined as +y axis on xz plane.
+//
+
+class Box : public Model {
+ public:
+  EIGEN_MAKE_ALIGNED_OPERATOR_NEW
+
+  explicit Box(const std::string& category);
+  ~Box() override = default;
+
+  bool InsideTest(const Vector3f& point, int check_axis) const;
+
+  const std::vector<Face>& GetFaces() const { return faces_; }
+  const Face& GetFace(size_t face_id) const { return faces_[face_id]; }
+
+  const std::vector<std::array<int, 2>>& GetEdges() const { return edges_; }
+  const std::array<int, 2>& GetEdge(size_t edge_id) const {
+    return edges_[edge_id];
+  }
+
+  // Returns the keypoints for the front face of the box.
+  // The front face is defind as a face with +z normal vector on xy plane
+  // In Box's c'tor, the top face is set to {1, 3, 7, 5}
+  const Face& GetFrontFace() const;
+
+  // Returns the keypoints for the top face of the box.
+  // The top face is defind as a face with +z normal vector on xy plane
+  // In Box's c'tor, the top face is set to {1, 3, 7, 5}
+  const Face& GetTopFace() const;
+
+  void Update() override;
+  void Adjust(const std::vector<float>& variables) override;
+  float* GetVertex(size_t vertex_id) override;
+  const float* GetVertex(size_t vertex_id) const override;
+  void Deserialize(const Object& obj) override;
+  void Serialize(Object* obj) override;
+
+  // Computes the plane center and the normal vector for the plane the object
+  // is sitting on in the world cooordinate system. The normal vector is roughly
+  // aligned with gravity.
+  std::pair<Vector3f, Vector3f> GetGroundPlane() const;
+
+  // Estimates a box 9-dof parameters from the given vertices. Directly computes
+  // the scale of the box, then solves for orientation and translation.
+  // Expects a std::vector of size 9 of a Eigen::Vector3f or mapped Vector3f.
+  // If mapping proto messages, we recommend to use the Map<const Vector3f>.
+  // For example:
+  //
+  // using T = Map<const Vector3f>;
+  // std::vector<T> vertices;
+  // for (const auto& point : message) { // point is a repeated float message.
+  //   T p(point.data());
+  //   vertices.emplace_back(p);
+  // }
+  // box.Fit<T>(vertices);
+  //
+  // The Points must be arranged as 1 + 8 (center keypoint followed by 8 box
+  // vertices) vector. This function will overwrite the scale and transformation
+  // properties of the class.
+  template <typename T = Eigen::Map<const Vector3f>>
+  void Fit(const std::vector<T>& vertices);
+
+ private:
+  std::vector<Face> faces_;
+  std::vector<std::array<int, 2>> edges_;
+  std::vector<Vector3f> bounding_box_;
+};
+
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_MODULES_OBJECTRON_CALCULATORS_BOX_H_

mediapipe/modules/objectron/calculators/box_util.cc

@@ -0,0 +1,153 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/modules/objectron/calculators/box_util.h"
+
+#include <math.h>
+
+#include "mediapipe/framework/port/logging.h"
+#include "mediapipe/framework/port/opencv_core_inc.h"
+#include "mediapipe/framework/port/opencv_imgproc_inc.h"
+#include "mediapipe/util/tracking/box_tracker.pb.h"
+
+namespace mediapipe {
+void ComputeBoundingRect(const std::vector<cv::Point2f>& points,
+                         mediapipe::TimedBoxProto* box) {
+  CHECK(box != nullptr);
+  float top = 1.0f;
+  float bottom = 0.0f;
+  float left = 1.0f;
+  float right = 0.0f;
+  for (const auto& point : points) {
+    top = std::min(top, point.y);
+    bottom = std::max(bottom, point.y);
+    left = std::min(left, point.x);
+    right = std::max(right, point.x);
+  }
+  box->set_top(top);
+  box->set_bottom(bottom);
+  box->set_left(left);
+  box->set_right(right);
+  // We are currently only doing axis aligned bounding box. If we need to
+  // compute rotated bounding box, then we need the original image aspect ratio,
+  // map back to original image space, compute cv::convexHull, then for each
+  // edge of the hull, rotate according to edge orientation, find the box.
+  box->set_rotation(0.0f);
+}
+
+float ComputeBoxIoU(const TimedBoxProto& box1, const TimedBoxProto& box2) {
+  cv::Point2f box1_center((box1.left() + box1.right()) * 0.5f,
+                          (box1.top() + box1.bottom()) * 0.5f);
+  cv::Size2f box1_size(box1.right() - box1.left(), box1.bottom() - box1.top());
+  cv::RotatedRect rect1(box1_center, box1_size,
+                        -box1.rotation() * 180.0f / M_PI);
+  cv::Point2f box2_center((box2.left() + box2.right()) * 0.5f,
+                          (box2.top() + box2.bottom()) * 0.5f);
+  cv::Size2f box2_size(box2.right() - box2.left(), box2.bottom() - box2.top());
+  cv::RotatedRect rect2(box2_center, box2_size,
+                        -box2.rotation() * 180.0f / M_PI);
+  std::vector<cv::Point2f> intersections_unsorted;
+  std::vector<cv::Point2f> intersections;
+  cv::rotatedRectangleIntersection(rect1, rect2, intersections_unsorted);
+  if (intersections_unsorted.size() < 3) {
+    return 0.0f;
+  }
+  cv::convexHull(intersections_unsorted, intersections);
+
+  // We use Shoelace formula to compute area of polygons.
+  float intersection_area = 0.0f;
+  for (int i = 0; i < intersections.size(); ++i) {
+    const auto& curr_pt = intersections[i];
+    const int i_next = (i + 1) == intersections.size() ? 0 : (i + 1);
+    const auto& next_pt = intersections[i_next];
+    intersection_area += (curr_pt.x * next_pt.y - next_pt.x * curr_pt.y);
+  }
+  intersection_area = std::abs(intersection_area) * 0.5f;
+
+  // Compute union area
+  const float union_area =
+      rect1.size.area() + rect2.size.area() - intersection_area + 1e-5f;
+
+  const float iou = intersection_area / union_area;
+  return iou;
+}
+
+std::vector<cv::Point2f> ComputeBoxCorners(const TimedBoxProto& box,
+                                           float width, float height) {
+  // Rotate 4 corner w.r.t. center.
+  const cv::Point2f center(0.5f * (box.left() + box.right()) * width,
+                           0.5f * (box.top() + box.bottom()) * height);
+  const std::vector<cv::Point2f> corners{
+      cv::Point2f(box.left() * width, box.top() * height),
+      cv::Point2f(box.left() * width, box.bottom() * height),
+      cv::Point2f(box.right() * width, box.bottom() * height),
+      cv::Point2f(box.right() * width, box.top() * height)};
+
+  const float cos_a = std::cos(box.rotation());
+  const float sin_a = std::sin(box.rotation());
+  std::vector<cv::Point2f> transformed_corners(4);
+  for (int k = 0; k < 4; ++k) {
+    // Scale and rotate w.r.t. center.
+    const cv::Point2f rad = corners[k] - center;
+    const cv::Point2f rot_rad(cos_a * rad.x - sin_a * rad.y,
+                              sin_a * rad.x + cos_a * rad.y);
+    transformed_corners[k] = center + rot_rad;
+    transformed_corners[k].x /= width;
+    transformed_corners[k].y /= height;
+  }
+  return transformed_corners;
+}
+
+cv::Mat PerspectiveTransformBetweenBoxes(const TimedBoxProto& src_box,
+                                         const TimedBoxProto& dst_box,
+                                         const float aspect_ratio) {
+  std::vector<cv::Point2f> box1_corners =
+      ComputeBoxCorners(src_box, /*width*/ aspect_ratio, /*height*/ 1.0f);
+  std::vector<cv::Point2f> box2_corners =
+      ComputeBoxCorners(dst_box, /*width*/ aspect_ratio, /*height*/ 1.0f);
+  cv::Mat affine_transform = cv::getPerspectiveTransform(
+      /*src*/ box1_corners, /*dst*/ box2_corners);
+  cv::Mat output_affine;
+  affine_transform.convertTo(output_affine, CV_32FC1);
+  return output_affine;
+}
+
+cv::Point2f MapPoint(const TimedBoxProto& src_box, const TimedBoxProto& dst_box,
+                     const cv::Point2f& src_point, float width, float height) {
+  const cv::Point2f src_center(
+      0.5f * (src_box.left() + src_box.right()) * width,
+      0.5f * (src_box.top() + src_box.bottom()) * height);
+  const cv::Point2f dst_center(
+      0.5f * (dst_box.left() + dst_box.right()) * width,
+      0.5f * (dst_box.top() + dst_box.bottom()) * height);
+  const float scale_x =
+      (dst_box.right() - dst_box.left()) / (src_box.right() - src_box.left());
+  const float scale_y =
+      (dst_box.bottom() - dst_box.top()) / (src_box.bottom() - src_box.top());
+  const float rotation = dst_box.rotation() - src_box.rotation();
+  const cv::Point2f rad =
+      cv::Point2f(src_point.x * width, src_point.y * height) - src_center;
+  const float rad_x = rad.x * scale_x;
+  const float rad_y = rad.y * scale_y;
+  const float cos_a = std::cos(rotation);
+  const float sin_a = std::sin(rotation);
+  const cv::Point2f rot_rad(cos_a * rad_x - sin_a * rad_y,
+                            sin_a * rad_x + cos_a * rad_y);
+  const cv::Point2f dst_point_image = dst_center + rot_rad;
+  const cv::Point2f dst_point(dst_point_image.x / width,
+                              dst_point_image.y / height);
+  return dst_point;
+}
+
+}  // namespace mediapipe

mediapipe/modules/objectron/calculators/box_util.h

@@ -0,0 +1,50 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_MODULES_OBJECTRON_CALCULATORS_BOX_UTIL_H_
+#define MEDIAPIPE_MODULES_OBJECTRON_CALCULATORS_BOX_UTIL_H_
+
+#include "mediapipe/framework/port/opencv_core_inc.h"
+#include "mediapipe/util/tracking/box_tracker.pb.h"
+
+namespace mediapipe {
+
+// This function fills the geometry of the TimedBoxProto. Id, timestamp etc.
+// need to be set outside this function.
+void ComputeBoundingRect(const std::vector<cv::Point2f>& points,
+                         mediapipe::TimedBoxProto* box);
+
+// This function computes the intersection over union between two boxes.
+float ComputeBoxIoU(const TimedBoxProto& box1, const TimedBoxProto& box2);
+
+// Computes corners of the box.
+// width and height are image width and height, which is typically
+// needed since the box is in normalized coordinates.
+std::vector<cv::Point2f> ComputeBoxCorners(const TimedBoxProto& box,
+                                           float width, float height);
+
+// Computes the perspective transform from box1 to box2.
+// The input argument aspect_ratio is width / height of the image.
+// The returned matrix should be a 3x3 matrix.
+cv::Mat PerspectiveTransformBetweenBoxes(const TimedBoxProto& src_box,
+                                         const TimedBoxProto& dst_box,
+                                         const float aspect_ratio);
+
+// Map point according to source and destination box location.
+cv::Point2f MapPoint(const TimedBoxProto& src_box, const TimedBoxProto& dst_box,
+                     const cv::Point2f& src_point, float width, float height);
+
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_MODULES_OBJECTRON_CALCULATORS_BOX_UTIL_H_

mediapipe/modules/objectron/calculators/box_util_test.cc

@@ -0,0 +1,123 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/modules/objectron/calculators/box_util.h"
+
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/opencv_core_inc.h"
+#include "mediapipe/util/tracking/box_tracker.pb.h"
+
+namespace mediapipe {
+namespace {
+
+TEST(BoxUtilTest, TestComputeBoundingRect) {
+  std::vector<cv::Point2f> points{
+      cv::Point2f(0.35f, 0.25f), cv::Point2f(0.3f, 0.3f),
+      cv::Point2f(0.2f, 0.4f),   cv::Point2f(0.3f, 0.1f),
+      cv::Point2f(0.2f, 0.2f),   cv::Point2f(0.5f, 0.3f),
+      cv::Point2f(0.4f, 0.4f),   cv::Point2f(0.5f, 0.1f),
+      cv::Point2f(0.4f, 0.2f)};
+  TimedBoxProto box;
+  ComputeBoundingRect(points, &box);
+  EXPECT_FLOAT_EQ(0.1f, box.top());
+  EXPECT_FLOAT_EQ(0.4f, box.bottom());
+  EXPECT_FLOAT_EQ(0.2f, box.left());
+  EXPECT_FLOAT_EQ(0.5f, box.right());
+}
+
+TEST(BoxUtilTest, TestComputeBoxIoU) {
+  TimedBoxProto box1;
+  box1.set_top(0.2f);
+  box1.set_bottom(0.6f);
+  box1.set_left(0.1f);
+  box1.set_right(0.3f);
+  box1.set_rotation(0.0f);
+  TimedBoxProto box2 = box1;
+  box2.set_rotation(/*pi/2*/ 1.570796f);
+  const float box_area =
+      (box1.bottom() - box1.top()) * (box1.right() - box1.left());
+  const float box_intersection =
+      (box1.right() - box1.left()) * (box1.right() - box1.left());
+  const float expected_iou =
+      box_intersection / (box_area * 2 - box_intersection);
+  EXPECT_NEAR(expected_iou, ComputeBoxIoU(box1, box2), 3e-5f);
+
+  TimedBoxProto box3;
+  box3.set_top(0.2f);
+  box3.set_bottom(0.6f);
+  box3.set_left(0.5f);
+  box3.set_right(0.7f);
+  EXPECT_NEAR(0.0f, ComputeBoxIoU(box1, box3), 3e-5f);
+}
+
+TEST(BoxUtilTest, TestPerspectiveTransformBetweenBoxes) {
+  TimedBoxProto box1;
+  const float height = 4.0f;
+  const float width = 3.0f;
+  box1.set_top(1.0f / height);
+  box1.set_bottom(2.0f / height);
+  box1.set_left(1.0f / width);
+  box1.set_right(2.0f / width);
+  TimedBoxProto box2;
+  box2.set_top(1.0f / height);
+  box2.set_bottom(2.0f / height);
+  box2.set_left(1.0f / width);
+  box2.set_right(2.0f / width);
+  box2.set_rotation(/*pi/4*/ -0.785398f);
+  cv::Mat transform =
+      PerspectiveTransformBetweenBoxes(box1, box2, width / height);
+  const float kTolerence = 1e-5f;
+  const cv::Vec3f original_position(1.5f / width, 1.0f / height, 1.0f);
+  const cv::Mat transformed_position = transform * cv::Mat(original_position);
+  EXPECT_NEAR(
+      (1.5f - 0.5f * std::sqrt(2) / 2.0f) / width,
+      transformed_position.at<float>(0) / transformed_position.at<float>(2),
+      kTolerence);
+  EXPECT_NEAR(
+      (1.5f - 0.5f * std::sqrt(2) / 2.0f) / height,
+      transformed_position.at<float>(1) / transformed_position.at<float>(2),
+      kTolerence);
+}
+
+TEST(BoxUtilTest, TestMapPoint) {
+  const float height = 4.0f;
+  const float width = 3.0f;
+  TimedBoxProto box1;
+  box1.set_top(1.0f / height);
+  box1.set_bottom(2.0f / height);
+  box1.set_left(1.0f / width);
+  box1.set_right(2.0f / width);
+  TimedBoxProto box2;
+  box2.set_top(1.0f / height);
+  box2.set_bottom(2.0f / height);
+  box2.set_left(1.0f / width);
+  box2.set_right(2.0f / width);
+  box2.set_rotation(/*pi/4*/ -0.785398f);
+
+  cv::Point2f src_point1(1.2f / width, 1.4f / height);
+  cv::Point2f src_point2(1.3f / width, 1.8f / height);
+  const float distance1 = std::sqrt(0.1 * 0.1 + 0.4 * 0.4);
+  cv::Point2f dst_point1 = MapPoint(box1, box2, src_point1, width, height);
+  cv::Point2f dst_point2 = MapPoint(box1, box2, src_point2, width, height);
+  const float distance2 =
+      std::sqrt((dst_point1.x * width - dst_point2.x * width) *
+                    (dst_point1.x * width - dst_point2.x * width) +
+                (dst_point1.y * height - dst_point2.y * height) *
+                    (dst_point1.y * height - dst_point2.y * height));
+  EXPECT_NEAR(distance1, distance2, 1e-5f);
+}
+
+}  // namespace
+}  // namespace mediapipe

mediapipe/modules/objectron/calculators/camera_parameters.proto

@@ -0,0 +1,47 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+message CameraParametersProto {
+  // This number is non-negative, it represents camera height above ground
+  // normalized by focal length.
+  optional float height_above_ground = 1 [default = 100.0];
+  // Width of image in portrait orientation normalized by focal length
+  optional float portrait_width = 2 [default = 1.0103];
+  // Height of image in portrait orientation normalized by focal length
+  optional float portrait_height = 3 [default = 1.3435];
+  enum ImageOrientation {
+    PORTRAIT_ORIENTATION = 0;
+    LANDSCAPE_ORIENTATION = 1;
+  }
+  // The input image orientation
+  optional ImageOrientation image_orientation = 4
+      [default = PORTRAIT_ORIENTATION];
+
+  // This defines the projection method from 2D screen to 3D.
+  enum ProjectionMode {
+    UNSPECIFIED = 0;
+    // Projects 2D point to ground plane (horizontal plane).
+    GROUND_PLANE = 1;
+    // Projects 2D point to sphere.
+    SPHERE = 2;
+  }
+  optional ProjectionMode projection_mode = 5 [default = GROUND_PLANE];
+  // Radius of sphere when using the SPHERE projection mode above.
+  // The value is normalized by focal length.
+  optional float projection_sphere_radius = 6 [default = 100.0];
+}

mediapipe/modules/objectron/calculators/decoder.cc

@@ -0,0 +1,257 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/modules/objectron/calculators/decoder.h"
+
+#include <limits>
+
+#include "Eigen/Dense"
+#include "mediapipe/framework/port/canonical_errors.h"
+#include "mediapipe/framework/port/logging.h"
+#include "mediapipe/framework/port/opencv_imgproc_inc.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/modules/objectron/calculators/annotation_data.pb.h"
+
+namespace mediapipe {
+constexpr int Decoder::kNumOffsetmaps = 16;
+
+namespace {
+void SetPoint3d(float x, float y, float z, Point3D* point_3d) {
+  point_3d->set_x(x);
+  point_3d->set_y(y);
+  point_3d->set_z(z);
+}
+}  // namespace
+
+FrameAnnotation Decoder::DecodeBoundingBoxKeypoints(
+    const cv::Mat& heatmap, const cv::Mat& offsetmap) const {
+  CHECK_EQ(1, heatmap.channels());
+  CHECK_EQ(kNumOffsetmaps, offsetmap.channels());
+  CHECK_EQ(heatmap.cols, offsetmap.cols);
+  CHECK_EQ(heatmap.rows, offsetmap.rows);
+
+  const float offset_scale = std::min(offsetmap.cols, offsetmap.rows);
+  const std::vector<cv::Point> center_points = ExtractCenterKeypoints(heatmap);
+  std::vector<BeliefBox> boxes;
+  for (const auto& center_point : center_points) {
+    BeliefBox box;
+    box.box_2d.emplace_back(center_point.x, center_point.y);
+    const int center_x = static_cast<int>(std::round(center_point.x));
+    const int center_y = static_cast<int>(std::round(center_point.y));
+    box.belief = heatmap.at<float>(center_y, center_x);
+    if (config_.voting_radius() > 1) {
+      DecodeByVoting(heatmap, offsetmap, center_x, center_y, offset_scale,
+                     offset_scale, &box);
+    } else {
+      DecodeByPeak(offsetmap, center_x, center_y, offset_scale, offset_scale,
+                   &box);
+    }
+    if (IsNewBox(&boxes, &box)) {
+      boxes.push_back(std::move(box));
+    }
+  }
+
+  const float x_scale = 1.0f / offsetmap.cols;
+  const float y_scale = 1.0f / offsetmap.rows;
+  FrameAnnotation frame_annotations;
+  for (const auto& box : boxes) {
+    auto* object = frame_annotations.add_annotations();
+    for (const auto& point : box.box_2d) {
+      auto* point2d = object->add_keypoints()->mutable_point_2d();
+      point2d->set_x(point.first * x_scale);
+      point2d->set_y(point.second * y_scale);
+    }
+  }
+  return frame_annotations;
+}
+
+void Decoder::DecodeByPeak(const cv::Mat& offsetmap, int center_x, int center_y,
+                           float offset_scale_x, float offset_scale_y,
+                           BeliefBox* box) const {
+  const auto& offset = offsetmap.at<cv::Vec<float, kNumOffsetmaps>>(
+      /*row*/ center_y, /*col*/ center_x);
+  for (int i = 0; i < kNumOffsetmaps / 2; ++i) {
+    const float x_offset = offset[2 * i] * offset_scale_x;
+    const float y_offset = offset[2 * i + 1] * offset_scale_y;
+    box->box_2d.emplace_back(center_x + x_offset, center_y + y_offset);
+  }
+}
+
+void Decoder::DecodeByVoting(const cv::Mat& heatmap, const cv::Mat& offsetmap,
+                             int center_x, int center_y, float offset_scale_x,
+                             float offset_scale_y, BeliefBox* box) const {
+  // Votes at the center.
+  const auto& center_offset = offsetmap.at<cv::Vec<float, kNumOffsetmaps>>(
+      /*row*/ center_y, /*col*/ center_x);
+  std::vector<float> center_votes(kNumOffsetmaps, 0.f);
+  for (int i = 0; i < kNumOffsetmaps / 2; ++i) {
+    center_votes[2 * i] = center_x + center_offset[2 * i] * offset_scale_x;
+    center_votes[2 * i + 1] =
+        center_y + center_offset[2 * i + 1] * offset_scale_y;
+  }
+
+  // Find voting window.
+  int x_min = std::max(0, center_x - config_.voting_radius());
+  int y_min = std::max(0, center_y - config_.voting_radius());
+  int width = std::min(heatmap.cols - x_min, config_.voting_radius() * 2 + 1);
+  int height = std::min(heatmap.rows - y_min, config_.voting_radius() * 2 + 1);
+  cv::Rect rect(x_min, y_min, width, height);
+  cv::Mat heat = heatmap(rect);
+  cv::Mat offset = offsetmap(rect);
+
+  for (int i = 0; i < kNumOffsetmaps / 2; ++i) {
+    float x_sum = 0.f;
+    float y_sum = 0.f;
+    float votes = 0.f;
+    for (int r = 0; r < heat.rows; ++r) {
+      for (int c = 0; c < heat.cols; ++c) {
+        const float belief = heat.at<float>(r, c);
+        if (belief < config_.voting_threshold()) {
+          continue;
+        }
+        float offset_x =
+            offset.at<cv::Vec<float, kNumOffsetmaps>>(r, c)[2 * i] *
+            offset_scale_x;
+        float offset_y =
+            offset.at<cv::Vec<float, kNumOffsetmaps>>(r, c)[2 * i + 1] *
+            offset_scale_y;
+        float vote_x = c + rect.x + offset_x;
+        float vote_y = r + rect.y + offset_y;
+        float x_diff = std::abs(vote_x - center_votes[2 * i]);
+        float y_diff = std::abs(vote_y - center_votes[2 * i + 1]);
+        if (x_diff > config_.voting_allowance() ||
+            y_diff > config_.voting_allowance()) {
+          continue;
+        }
+        x_sum += vote_x * belief;
+        y_sum += vote_y * belief;
+        votes += belief;
+      }
+    }
+    box->box_2d.emplace_back(x_sum / votes, y_sum / votes);
+  }
+}
+
+bool Decoder::IsNewBox(std::vector<BeliefBox>* boxes, BeliefBox* box) const {
+  for (auto& b : *boxes) {
+    if (IsIdentical(b, *box)) {
+      if (b.belief < box->belief) {
+        std::swap(b, *box);
+      }
+      return false;
+    }
+  }
+  return true;
+}
+
+bool Decoder::IsIdentical(const BeliefBox& box_1,
+                          const BeliefBox& box_2) const {
+  // Skip the center point.
+  for (int i = 1; i < box_1.box_2d.size(); ++i) {
+    const float x_diff =
+        std::abs(box_1.box_2d[i].first - box_2.box_2d[i].first);
+    const float y_diff =
+        std::abs(box_1.box_2d[i].second - box_2.box_2d[i].second);
+    if (x_diff > config_.voting_allowance() ||
+        y_diff > config_.voting_allowance()) {
+      return false;
+    }
+  }
+  return true;
+}
+
+std::vector<cv::Point> Decoder::ExtractCenterKeypoints(
+    const cv::Mat& center_heatmap) const {
+  cv::Mat max_filtered_heatmap(center_heatmap.rows, center_heatmap.cols,
+                               center_heatmap.type());
+  const int kernel_size =
+      static_cast<int>(config_.local_max_distance() * 2 + 1 + 0.5f);
+  const cv::Size morph_size(kernel_size, kernel_size);
+  cv::dilate(center_heatmap, max_filtered_heatmap,
+             cv::getStructuringElement(cv::MORPH_RECT, morph_size));
+  cv::Mat peak_map;
+  cv::bitwise_and((center_heatmap >= max_filtered_heatmap),
+                  (center_heatmap >= config_.heatmap_threshold()), peak_map);
+  std::vector<cv::Point> locations;  // output, locations of non-zero pixels
+  cv::findNonZero(peak_map, locations);
+  return locations;
+}
+
+absl::Status Decoder::Lift2DTo3D(
+    const Eigen::Matrix<float, 4, 4, Eigen::RowMajor>& projection_matrix,
+    bool portrait, FrameAnnotation* estimated_box) const {
+  CHECK(estimated_box != nullptr);
+  const float fx = projection_matrix(0, 0);
+  const float fy = projection_matrix(1, 1);
+  const float cx = projection_matrix(0, 2);
+  const float cy = projection_matrix(1, 2);
+  for (auto& annotation : *estimated_box->mutable_annotations()) {
+    Eigen::Matrix<float, 16, 12, Eigen::RowMajor> m =
+        Eigen::Matrix<float, 16, 12, Eigen::RowMajor>::Zero(16, 12);
+    CHECK_EQ(9, annotation.keypoints_size());
+    float u, v;
+    for (int i = 0; i < 8; ++i) {
+      const auto& keypoint2d = annotation.keypoints(i + 1).point_2d();
+      if (portrait) {
+        // swap x and y given that our image is in portrait orientation
+        u = keypoint2d.y() * 2 - 1;
+        v = keypoint2d.x() * 2 - 1;
+      } else {
+        u = keypoint2d.x() * 2 - 1;
+        v = 1 - keypoint2d.y() * 2;  // (1 - keypoint2d.y()) * 2 - 1
+      }
+      for (int j = 0; j < 4; ++j) {
+        // For each of the 4 control points, formulate two rows of the
+        // m matrix (two equations).
+        const float control_alpha = epnp_alpha_(i, j);
+        m(i * 2, j * 3) = fx * control_alpha;
+        m(i * 2, j * 3 + 2) = (cx + u) * control_alpha;
+        m(i * 2 + 1, j * 3 + 1) = fy * control_alpha;
+        m(i * 2 + 1, j * 3 + 2) = (cy + v) * control_alpha;
+      }
+    }
+    // This is a self adjoint matrix. Use SelfAdjointEigenSolver for a fast
+    // and stable solution.
+    Eigen::Matrix<float, 12, 12, Eigen::RowMajor> mt_m = m.transpose() * m;
+    Eigen::SelfAdjointEigenSolver<Eigen::Matrix<float, 12, 12, Eigen::RowMajor>>
+        eigen_solver(mt_m);
+    if (eigen_solver.info() != Eigen::Success) {
+      return absl::AbortedError("Eigen decomposition failed.");
+    }
+    CHECK_EQ(12, eigen_solver.eigenvalues().size());
+    // Eigenvalues are sorted in increasing order for SelfAdjointEigenSolver
+    // only! If you use other Eigen Solvers, it's not guaranteed to be in
+    // increasing order. Here, we just take the eigen vector corresponding
+    // to first/smallest eigen value, since we used SelfAdjointEigenSolver.
+    Eigen::VectorXf eigen_vec = eigen_solver.eigenvectors().col(0);
+    Eigen::Map<Eigen::Matrix<float, 4, 3, Eigen::RowMajor>> control_matrix(
+        eigen_vec.data());
+    if (control_matrix(0, 2) > 0) {
+      control_matrix = -control_matrix;
+    }
+    // First set the center keypoint.
+    SetPoint3d(control_matrix(0, 0), control_matrix(0, 1), control_matrix(0, 2),
+               annotation.mutable_keypoints(0)->mutable_point_3d());
+    // Then set the 8 vertices.
+    Eigen::Matrix<float, 8, 3, Eigen::RowMajor> vertices =
+        epnp_alpha_ * control_matrix;
+    for (int i = 0; i < 8; ++i) {
+      SetPoint3d(vertices(i, 0), vertices(i, 1), vertices(i, 2),
+                 annotation.mutable_keypoints(i + 1)->mutable_point_3d());
+    }
+  }
+  return absl::OkStatus();
+}
+
+}  // namespace mediapipe

mediapipe/modules/objectron/calculators/decoder.h

@@ -0,0 +1,109 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_MODULES_OBJECTRON_CALCULATORS_DECODER_H_
+#define MEDIAPIPE_MODULES_OBJECTRON_CALCULATORS_DECODER_H_
+
+#include <vector>
+
+#include "Eigen/Dense"
+#include "absl/status/status.h"
+#include "mediapipe/framework/port/opencv_core_inc.h"
+#include "mediapipe/modules/objectron/calculators/annotation_data.pb.h"
+#include "mediapipe/modules/objectron/calculators/belief_decoder_config.pb.h"
+
+namespace mediapipe {
+
+// Decodes 3D bounding box from heatmaps and offset maps. In the future,
+// if we want to develop decoder for generic skeleton, then we need to
+// generalize this class, and make a few child classes.
+class Decoder {
+ public:
+  static const int kNumOffsetmaps;
+
+  explicit Decoder(const BeliefDecoderConfig& config) : config_(config) {
+    epnp_alpha_ << 4.0f, -1.0f, -1.0f, -1.0f, 2.0f, -1.0f, -1.0f, 1.0f, 2.0f,
+        -1.0f, 1.0f, -1.0f, 0.0f, -1.0f, 1.0f, 1.0f, 2.0f, 1.0f, -1.0f, -1.0f,
+        0.0f, 1.0f, -1.0f, 1.0f, 0.0f, 1.0f, 1.0f, -1.0f, -2.0f, 1.0f, 1.0f,
+        1.0f;
+  }
+
+  // Decodes bounding boxes from predicted heatmap and offset maps.
+  // Input:
+  //   heatmap: a single channel cv::Mat representing center point heatmap
+  //   offsetmap: a 16 channel cv::Mat representing the 16 offset maps
+  //              (2 for each of the 8 vertices)
+  // Output:
+  //   Outputs 3D bounding boxes 2D vertices, represented by 'point_2d' field
+  //   in each 'keypoints' field of object annotations.
+  FrameAnnotation DecodeBoundingBoxKeypoints(const cv::Mat& heatmap,
+                                             const cv::Mat& offsetmap) const;
+
+  // Lifts the estimated 2D projections of bounding box vertices to 3D.
+  // This function uses the EPnP approach described in this paper:
+  // https://icwww.epfl.ch/~lepetit/papers/lepetit_ijcv08.pdf .
+  // Input:
+  //   projection_matrix: the projection matrix from 3D coordinate
+  //     to screen coordinate.
+  //     The 2D screen coordinate is defined as: u is along the long
+  //     edge of the device, pointing down; v is along the short edge
+  //     of the device, pointing right.
+  //   portrait: a boolen variable indicating whether our images are
+  //     obtained in portrait orientation or not.
+  //   estimated_box: annotation with point_2d field populated with
+  //     2d vertices.
+  // Output:
+  //   estimated_box: annotation with point_3d field populated with
+  //     3d vertices.
+  absl::Status Lift2DTo3D(
+      const Eigen::Matrix<float, 4, 4, Eigen::RowMajor>& projection_matrix,
+      bool portrait, FrameAnnotation* estimated_box) const;
+
+ private:
+  struct BeliefBox {
+    float belief;
+    std::vector<std::pair<float, float>> box_2d;
+  };
+
+  std::vector<cv::Point> ExtractCenterKeypoints(
+      const cv::Mat& center_heatmap) const;
+
+  // Decodes 2D keypoints at the peak point.
+  void DecodeByPeak(const cv::Mat& offsetmap, int center_x, int center_y,
+                    float offset_scale_x, float offset_scale_y,
+                    BeliefBox* box) const;
+
+  // Decodes 2D keypoints by voting around the peak.
+  void DecodeByVoting(const cv::Mat& heatmap, const cv::Mat& offsetmap,
+                      int center_x, int center_y, float offset_scale_x,
+                      float offset_scale_y, BeliefBox* box) const;
+
+  // Returns true if it is a new box. Otherwise, it may replace an existing box
+  // if the new box's belief is higher.
+  bool IsNewBox(std::vector<BeliefBox>* boxes, BeliefBox* box) const;
+
+  // Returns true if the two boxes are identical.
+  bool IsIdentical(const BeliefBox& box_1, const BeliefBox& box_2) const;
+
+  BeliefDecoderConfig config_;
+  // Following equation (1) in this paper
+  // https://icwww.epfl.ch/~lepetit/papers/lepetit_ijcv08.pdf,
+  // this variable denotes the coefficients for the 4 control points
+  // for each of the 8 3D box vertices.
+  Eigen::Matrix<float, 8, 4, Eigen::RowMajor> epnp_alpha_;
+};
+
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_MODULES_OBJECTRON_CALCULATORS_DECODER_H_

mediapipe/modules/objectron/calculators/filter_detection_calculator.cc

@@ -0,0 +1,265 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "absl/container/node_hash_set.h"
+#include "absl/strings/str_split.h"
+#include "absl/strings/string_view.h"
+#include "absl/strings/strip.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/detection.pb.h"
+#include "mediapipe/framework/formats/location_data.pb.h"
+#include "mediapipe/framework/port/logging.h"
+#include "mediapipe/framework/port/map_util.h"
+#include "mediapipe/framework/port/re2.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/modules/objectron/calculators/filter_detection_calculator.pb.h"
+
+namespace mediapipe {
+
+namespace {
+
+constexpr char kDetectionTag[] = "DETECTION";
+constexpr char kDetectionsTag[] = "DETECTIONS";
+constexpr char kLabelsTag[] = "LABELS";
+constexpr char kLabelsCsvTag[] = "LABELS_CSV";
+
+using mediapipe::ContainsKey;
+using mediapipe::RE2;
+using Detections = std::vector<Detection>;
+using Strings = std::vector<std::string>;
+
+}  // namespace
+
+// Filters the entries in a Detection to only those with valid scores
+// for the specified allowed labels. Allowed labels are provided as a
+// vector<std::string> in an optional input side packet. Allowed labels can
+// contain simple strings or regular expressions. The valid score range
+// can be set in the options.The allowed labels can be provided as
+// vector<std::string> (LABELS) or CSV std::string (LABELS_CSV) containing class
+// names of allowed labels. Note: Providing an empty vector in the input side
+// packet Packet causes this calculator to act as a sink if
+// empty_allowed_labels_means_allow_everything is set to false (default value).
+// To allow all labels, use the calculator with no input side packet stream, or
+// set empty_allowed_labels_means_allow_everything to true.
+//
+// Example config:
+// node {
+//   calculator: "FilterDetectionCalculator"
+//   input_stream: "DETECTIONS:detections"
+//   output_stream: "DETECTIONS:filtered_detections"
+//   input_side_packet: "LABELS:allowed_labels"
+//   options: {
+//     [mediapipe.FilterDetectionCalculatorOptions.ext]: {
+//       min_score: 0.5
+//     }
+//   }
+// }
+
+struct FirstGreaterComparator {
+  bool operator()(const std::pair<float, int>& a,
+                  const std::pair<float, int>& b) const {
+    return a.first > b.first;
+  }
+};
+
+mediapipe::Status SortLabelsByDecreasingScore(const Detection& detection,
+                                              Detection* sorted_detection) {
+  RET_CHECK(sorted_detection);
+  RET_CHECK_EQ(detection.score_size(), detection.label_size());
+  if (!detection.label_id().empty()) {
+    RET_CHECK_EQ(detection.score_size(), detection.label_id_size());
+  }
+  // Copies input to keep all fields unchanged, and to reserve space for
+  // repeated fields. Repeated fields (score, label, and label_id) will be
+  // overwritten.
+  *sorted_detection = detection;
+
+  std::vector<std::pair<float, int>> scores_and_indices(detection.score_size());
+  for (int i = 0; i < detection.score_size(); ++i) {
+    scores_and_indices[i].first = detection.score(i);
+    scores_and_indices[i].second = i;
+  }
+
+  std::sort(scores_and_indices.begin(), scores_and_indices.end(),
+            FirstGreaterComparator());
+
+  for (int i = 0; i < detection.score_size(); ++i) {
+    const int index = scores_and_indices[i].second;
+    sorted_detection->set_score(i, detection.score(index));
+    sorted_detection->set_label(i, detection.label(index));
+  }
+
+  if (!detection.label_id().empty()) {
+    for (int i = 0; i < detection.score_size(); ++i) {
+      const int index = scores_and_indices[i].second;
+      sorted_detection->set_label_id(i, detection.label_id(index));
+    }
+  }
+  return mediapipe::OkStatus();
+}
+
+class FilterDetectionCalculator : public CalculatorBase {
+ public:
+  static mediapipe::Status GetContract(CalculatorContract* cc);
+  mediapipe::Status Open(CalculatorContext* cc) override;
+  mediapipe::Status Process(CalculatorContext* cc) override;
+
+ private:
+  bool IsValidLabel(const std::string& label);
+  bool IsValidScore(float score);
+  // Stores numeric limits for filtering on the score.
+  FilterDetectionCalculatorOptions options_;
+  // We use the next two fields to possibly filter to a limited set of
+  // classes.  The hash_set will be empty in two cases: 1) if no input
+  // side packet stream is provided (not filtering on labels), or 2)
+  // if the input side packet contains an empty vector (no labels are
+  // allowed). We use limit_labels_ to distinguish between the two cases.
+  bool limit_labels_ = true;
+  absl::node_hash_set<std::string> allowed_labels_;
+};
+REGISTER_CALCULATOR(FilterDetectionCalculator);
+
+mediapipe::Status FilterDetectionCalculator::GetContract(
+    CalculatorContract* cc) {
+  RET_CHECK(!cc->Inputs().GetTags().empty());
+  RET_CHECK(!cc->Outputs().GetTags().empty());
+
+  if (cc->Inputs().HasTag(kDetectionTag)) {
+    cc->Inputs().Tag(kDetectionTag).Set<Detection>();
+    cc->Outputs().Tag(kDetectionTag).Set<Detection>();
+  }
+  if (cc->Inputs().HasTag(kDetectionsTag)) {
+    cc->Inputs().Tag(kDetectionsTag).Set<Detections>();
+    cc->Outputs().Tag(kDetectionsTag).Set<Detections>();
+  }
+  if (cc->InputSidePackets().HasTag(kLabelsTag)) {
+    cc->InputSidePackets().Tag(kLabelsTag).Set<Strings>();
+  }
+  if (cc->InputSidePackets().HasTag(kLabelsCsvTag)) {
+    cc->InputSidePackets().Tag(kLabelsCsvTag).Set<std::string>();
+  }
+  return mediapipe::OkStatus();
+}
+
+mediapipe::Status FilterDetectionCalculator::Open(CalculatorContext* cc) {
+  cc->SetOffset(TimestampDiff(0));
+  options_ = cc->Options<FilterDetectionCalculatorOptions>();
+  limit_labels_ = cc->InputSidePackets().HasTag(kLabelsTag) ||
+                  cc->InputSidePackets().HasTag(kLabelsCsvTag);
+  if (limit_labels_) {
+    Strings whitelist_labels;
+    if (cc->InputSidePackets().HasTag(kLabelsCsvTag)) {
+      whitelist_labels = absl::StrSplit(
+          cc->InputSidePackets().Tag(kLabelsCsvTag).Get<std::string>(), ',',
+          absl::SkipWhitespace());
+      for (auto& e : whitelist_labels) {
+        absl::StripAsciiWhitespace(&e);
+      }
+    } else {
+      whitelist_labels = cc->InputSidePackets().Tag(kLabelsTag).Get<Strings>();
+    }
+    allowed_labels_.insert(whitelist_labels.begin(), whitelist_labels.end());
+  }
+  if (limit_labels_ && allowed_labels_.empty()) {
+    if (options_.fail_on_empty_labels()) {
+      cc->GetCounter("VideosWithEmptyLabelsWhitelist")->Increment();
+      return tool::StatusFail(
+          "FilterDetectionCalculator received empty whitelist with "
+          "fail_on_empty_labels = true.");
+    }
+    if (options_.empty_allowed_labels_means_allow_everything()) {
+      // Continue as if side_input was not provided, i.e. pass all labels.
+      limit_labels_ = false;
+    }
+  }
+  return mediapipe::OkStatus();
+}
+
+mediapipe::Status FilterDetectionCalculator::Process(CalculatorContext* cc) {
+  if (limit_labels_ && allowed_labels_.empty()) {
+    return mediapipe::OkStatus();
+  }
+  Detections detections;
+  if (cc->Inputs().HasTag(kDetectionsTag)) {
+    detections = cc->Inputs().Tag(kDetectionsTag).Get<Detections>();
+  } else if (cc->Inputs().HasTag(kDetectionTag)) {
+    detections.emplace_back(cc->Inputs().Tag(kDetectionsTag).Get<Detection>());
+  }
+  std::unique_ptr<Detections> outputs(new Detections);
+  for (const auto& input : detections) {
+    Detection output;
+    for (int i = 0; i < input.label_size(); ++i) {
+      const std::string& label = input.label(i);
+      const float score = input.score(i);
+      if (IsValidLabel(label) && IsValidScore(score)) {
+        output.add_label(label);
+        output.add_score(score);
+      }
+    }
+    if (output.label_size() > 0) {
+      if (input.has_location_data()) {
+        *output.mutable_location_data() = input.location_data();
+      }
+      Detection output_sorted;
+      if (!SortLabelsByDecreasingScore(output, &output_sorted).ok()) {
+        // Uses the orginal output if fails to sort.
+        cc->GetCounter("FailedToSortLabelsInDetection")->Increment();
+        output_sorted = output;
+      }
+      outputs->emplace_back(output_sorted);
+    }
+  }
+
+  if (cc->Outputs().HasTag(kDetectionsTag)) {
+    cc->Outputs()
+        .Tag(kDetectionsTag)
+        .Add(outputs.release(), cc->InputTimestamp());
+  } else if (!outputs->empty()) {
+    cc->Outputs()
+        .Tag(kDetectionsTag)
+        .Add(new Detection((*outputs)[0]), cc->InputTimestamp());
+  }
+  return mediapipe::OkStatus();
+}
+
+bool FilterDetectionCalculator::IsValidLabel(const std::string& label) {
+  bool match = !limit_labels_ || ContainsKey(allowed_labels_, label);
+  if (!match) {
+    // If no exact match is found, check for regular expression
+    // comparions in the allowed_labels.
+    for (const auto& label_regexp : allowed_labels_) {
+      match = match || RE2::FullMatch(label, RE2(label_regexp));
+    }
+  }
+  return match;
+}
+
+bool FilterDetectionCalculator::IsValidScore(float score) {
+  if (options_.has_min_score() && score < options_.min_score()) {
+    LOG(ERROR) << "Filter out detection with low score " << score;
+    return false;
+  }
+  if (options_.has_max_score() && score > options_.max_score()) {
+    LOG(ERROR) << "Filter out detection with high score " << score;
+    return false;
+  }
+  LOG(ERROR) << "Pass detection with score " << score;
+  return true;
+}
+
+}  // namespace mediapipe

mediapipe/modules/objectron/calculators/filter_detection_calculator.proto

@@ -0,0 +1,45 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+
+message FilterDetectionCalculatorOptions {
+  extend CalculatorOptions {
+    optional FilterDetectionCalculatorOptions ext = 339582987;
+  }
+  optional float min_score = 1;
+  optional float max_score = 2;
+  // Setting fail_on_empty_labels to true will cause the calculator to return a
+  // failure status on Open() if an empty list is provided on the external
+  // input, immediately terminating the graph run.
+  optional bool fail_on_empty_labels = 3 [default = false];
+  // If fail_on_empty_labels is set to false setting
+  // empty_allowed_labels_means_allow_everything to
+  // false will cause the calculator to close output stream and ignore remaining
+  // inputs if an empty list is provided. If
+  // empty_allowed_labels_means_allow_everything is set to true this will force
+  // calculator to pass all labels.
+  optional bool empty_allowed_labels_means_allow_everything = 6
+      [default = false];
+  // Determines whether the input format is a vector<Detection> (use-case object
+  // detectors) or Detection (use-case classifiers).
+  optional bool use_detection_vector = 4 [deprecated = true];
+  // Determines whether the input side packet format is a vector of labels, or
+  // a string with comma separated labels.
+  optional bool use_allowed_labels_csv = 5 [deprecated = true];
+}

mediapipe/modules/objectron/calculators/frame_annotation_to_rect_calculator.cc

@@ -0,0 +1,185 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+
+#include <cmath>
+
+#include "Eigen/Dense"
+#include "absl/memory/memory.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/rect.pb.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/modules/objectron/calculators/annotation_data.pb.h"
+#include "mediapipe/modules/objectron/calculators/box.h"
+#include "mediapipe/modules/objectron/calculators/frame_annotation_to_rect_calculator.pb.h"
+
+namespace mediapipe {
+
+using Matrix3fRM = Eigen::Matrix<float, 3, 3, Eigen::RowMajor>;
+using Eigen::Vector2f;
+using Eigen::Vector3f;
+
+namespace {
+
+constexpr char kInputFrameAnnotationTag[] = "FRAME_ANNOTATION";
+constexpr char kOutputNormRectTag[] = "NORM_RECT";
+
+}  // namespace
+
+// A calculator that converts FrameAnnotation proto to NormalizedRect.
+// The rotation angle of the NormalizedRect is derived from object's 3d pose.
+// The angle is calculated such that after rotation the 2d projection of y-axis.
+// on the image plane is always vertical.
+class FrameAnnotationToRectCalculator : public CalculatorBase {
+ public:
+  enum ViewStatus {
+    TOP_VIEW_ON,
+    TOP_VIEW_OFF,
+  };
+
+  static ::mediapipe::Status GetContract(CalculatorContract* cc);
+  ::mediapipe::Status Open(CalculatorContext* cc) override;
+  ::mediapipe::Status Process(CalculatorContext* cc) override;
+
+ private:
+  void AnnotationToRect(const FrameAnnotation& annotation,
+                        NormalizedRect* rect);
+  float RotationAngleFromAnnotation(const FrameAnnotation& annotation);
+
+  float RotationAngleFromPose(const Matrix3fRM& rotation,
+                              const Vector3f& translation, const Vector3f& vec);
+  ViewStatus status_;
+  float off_threshold_;
+  float on_threshold_;
+};
+REGISTER_CALCULATOR(FrameAnnotationToRectCalculator);
+
+::mediapipe::Status FrameAnnotationToRectCalculator::Open(
+    CalculatorContext* cc) {
+  status_ = TOP_VIEW_OFF;
+  const auto& options = cc->Options<FrameAnnotationToRectCalculatorOptions>();
+  off_threshold_ = options.off_threshold();
+  on_threshold_ = options.on_threshold();
+  RET_CHECK(off_threshold_ <= on_threshold_);
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status FrameAnnotationToRectCalculator::GetContract(
+    CalculatorContract* cc) {
+  RET_CHECK(!cc->Inputs().GetTags().empty());
+  RET_CHECK(!cc->Outputs().GetTags().empty());
+
+  if (cc->Inputs().HasTag(kInputFrameAnnotationTag)) {
+    cc->Inputs().Tag(kInputFrameAnnotationTag).Set<FrameAnnotation>();
+  }
+
+  if (cc->Outputs().HasTag(kOutputNormRectTag)) {
+    cc->Outputs().Tag(kOutputNormRectTag).Set<NormalizedRect>();
+  }
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status FrameAnnotationToRectCalculator::Process(
+    CalculatorContext* cc) {
+  if (cc->Inputs().Tag(kInputFrameAnnotationTag).IsEmpty()) {
+    return ::mediapipe::OkStatus();
+  }
+  auto output_rect = absl::make_unique<NormalizedRect>();
+  AnnotationToRect(
+      cc->Inputs().Tag(kInputFrameAnnotationTag).Get<FrameAnnotation>(),
+      output_rect.get());
+
+  // Output
+  cc->Outputs()
+      .Tag(kOutputNormRectTag)
+      .Add(output_rect.release(), cc->InputTimestamp());
+  return ::mediapipe::OkStatus();
+}
+
+void FrameAnnotationToRectCalculator::AnnotationToRect(
+    const FrameAnnotation& annotation, NormalizedRect* rect) {
+  float x_min = std::numeric_limits<float>::max();
+  float x_max = std::numeric_limits<float>::min();
+  float y_min = std::numeric_limits<float>::max();
+  float y_max = std::numeric_limits<float>::min();
+  const auto& object = annotation.annotations(0);
+  for (const auto& keypoint : object.keypoints()) {
+    const auto& point_2d = keypoint.point_2d();
+    x_min = std::min(x_min, point_2d.x());
+    x_max = std::max(x_max, point_2d.x());
+    y_min = std::min(y_min, point_2d.y());
+    y_max = std::max(y_max, point_2d.y());
+  }
+  rect->set_x_center((x_min + x_max) / 2);
+  rect->set_y_center((y_min + y_max) / 2);
+  rect->set_width(x_max - x_min);
+  rect->set_height(y_max - y_min);
+  rect->set_rotation(RotationAngleFromAnnotation(annotation));
+}
+
+float FrameAnnotationToRectCalculator::RotationAngleFromAnnotation(
+    const FrameAnnotation& annotation) {
+  const auto& object = annotation.annotations(0);
+  Box box("category");
+  std::vector<Vector3f> vertices_3d;
+  std::vector<Vector2f> vertices_2d;
+  for (const auto& keypoint : object.keypoints()) {
+    const auto& point_3d = keypoint.point_3d();
+    const auto& point_2d = keypoint.point_2d();
+    vertices_3d.emplace_back(
+        Vector3f(point_3d.x(), point_3d.y(), point_3d.z()));
+    vertices_2d.emplace_back(Vector2f(point_2d.x(), point_2d.y()));
+  }
+  box.Fit(vertices_3d);
+  Vector3f scale = box.GetScale();
+  Matrix3fRM box_rotation = box.GetRotation();
+  Vector3f box_translation = box.GetTranslation();
+
+  // Rotation angle to use when top-view is on(top-view on),
+  // Which will make z-axis upright after the rotation.
+  const float angle_on =
+      RotationAngleFromPose(box_rotation, box_translation, Vector3f::UnitZ());
+  // Rotation angle to use when side-view is on(top-view off),
+  // Which will make y-axis upright after the rotation.
+  const float angle_off =
+      RotationAngleFromPose(box_rotation, box_translation, Vector3f::UnitY());
+
+  // Calculate angle between z-axis and viewing ray in degrees.
+  const float view_to_z_angle = std::acos(box_rotation(2, 1)) * 180 / M_PI;
+
+  // Determine threshold based on current status,
+  // on_threshold_ is used for TOP_VIEW_ON -> TOP_VIEW_OFF transition,
+  // off_threshold_ is used for TOP_VIEW_OFF -> TOP_VIEW_ON transition.
+  const float thresh =
+      (status_ == TOP_VIEW_ON) ? on_threshold_ : off_threshold_;
+
+  // If view_to_z_angle is smaller than threshold, then top-view is on;
+  // Otherwise top-view is off.
+  status_ = (view_to_z_angle < thresh) ? TOP_VIEW_ON : TOP_VIEW_OFF;
+
+  // Determine which angle to used based on current status_.
+  float angle_to_rotate = (status_ == TOP_VIEW_ON) ? angle_on : angle_off;
+  return angle_to_rotate;
+}
+
+float FrameAnnotationToRectCalculator::RotationAngleFromPose(
+    const Matrix3fRM& rotation, const Vector3f& translation,
+    const Vector3f& vec) {
+  auto p1 = rotation * vec + translation;
+  auto p2 = -rotation * vec + translation;
+  const float dy = p2[2] * p1[1] - p1[2] * p2[1];
+  const float dx = p2[2] * p1[0] - p1[2] * p2[0];
+  return std::atan2(-dy, dx);
+}
+
+}  // namespace mediapipe

mediapipe/modules/objectron/calculators/frame_annotation_to_rect_calculator.proto

@@ -0,0 +1,31 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+
+message FrameAnnotationToRectCalculatorOptions {
+  extend CalculatorOptions {
+    optional FrameAnnotationToRectCalculatorOptions ext = 338119067;
+  }
+
+  // The threshold to use when top-view is off,to enable hysteresis,
+  // It's required that off_threshold <= on_threshold.
+  optional float off_threshold = 1 [default = 40.0];
+  // The threshold to use when top-view is on.
+  optional float on_threshold = 2 [default = 41.0];
+}

mediapipe/modules/objectron/calculators/frame_annotation_to_timed_box_list_calculator.cc

@@ -0,0 +1,115 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <utility>
+
+#include "absl/memory/memory.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/port/opencv_core_inc.h"
+#include "mediapipe/framework/port/opencv_imgproc_inc.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/modules/objectron/calculators/annotation_data.pb.h"
+#include "mediapipe/modules/objectron/calculators/box_util.h"
+#include "mediapipe/util/tracking/box_tracker.pb.h"
+
+namespace {
+constexpr char kInputStreamTag[] = "FRAME_ANNOTATION";
+constexpr char kOutputStreamTag[] = "BOXES";
+}  // namespace
+
+namespace mediapipe {
+
+// Convert FrameAnnotation 3d bounding box detections to TimedBoxListProto
+// 2d bounding boxes.
+//
+// Input:
+//  FRAME_ANNOTATION - 3d bounding box annotation.
+// Output:
+//  BOXES - 2d bounding box enclosing the projection of 3d box.
+//
+// Usage example:
+// node {
+//   calculator: "FrameAnnotationToTimedBoxListCalculator"
+//   input_stream: "FRAME_ANNOTATION:frame_annotation"
+//   output_stream: "BOXES:boxes"
+// }
+class FrameAnnotationToTimedBoxListCalculator : public CalculatorBase {
+ public:
+  static mediapipe::Status GetContract(CalculatorContract* cc);
+
+  mediapipe::Status Open(CalculatorContext* cc) override;
+  mediapipe::Status Process(CalculatorContext* cc) override;
+  mediapipe::Status Close(CalculatorContext* cc) override;
+};
+REGISTER_CALCULATOR(FrameAnnotationToTimedBoxListCalculator);
+
+mediapipe::Status FrameAnnotationToTimedBoxListCalculator::GetContract(
+    CalculatorContract* cc) {
+  RET_CHECK(!cc->Inputs().GetTags().empty());
+  RET_CHECK(!cc->Outputs().GetTags().empty());
+
+  if (cc->Inputs().HasTag(kInputStreamTag)) {
+    cc->Inputs().Tag(kInputStreamTag).Set<FrameAnnotation>();
+  }
+
+  if (cc->Outputs().HasTag(kOutputStreamTag)) {
+    cc->Outputs().Tag(kOutputStreamTag).Set<TimedBoxProtoList>();
+  }
+  return mediapipe::OkStatus();
+}
+
+mediapipe::Status FrameAnnotationToTimedBoxListCalculator::Open(
+    CalculatorContext* cc) {
+  return mediapipe::OkStatus();
+}
+
+mediapipe::Status FrameAnnotationToTimedBoxListCalculator::Process(
+    CalculatorContext* cc) {
+  if (cc->Inputs().HasTag(kInputStreamTag) &&
+      !cc->Inputs().Tag(kInputStreamTag).IsEmpty()) {
+    const auto& frame_annotation =
+        cc->Inputs().Tag(kInputStreamTag).Get<FrameAnnotation>();
+    auto output_objects = absl::make_unique<TimedBoxProtoList>();
+    for (const auto& annotation : frame_annotation.annotations()) {
+      std::vector<cv::Point2f> key_points;
+      for (const auto& keypoint : annotation.keypoints()) {
+        key_points.push_back(
+            cv::Point2f(keypoint.point_2d().x(), keypoint.point_2d().y()));
+      }
+      TimedBoxProto* added_box = output_objects->add_box();
+      ComputeBoundingRect(key_points, added_box);
+      added_box->set_id(annotation.object_id());
+      const int64 time_msec =
+          static_cast<int64>(std::round(frame_annotation.timestamp() / 1000));
+      added_box->set_time_msec(time_msec);
+    }
+
+    // Output
+    if (cc->Outputs().HasTag(kOutputStreamTag)) {
+      cc->Outputs()
+          .Tag(kOutputStreamTag)
+          .Add(output_objects.release(), cc->InputTimestamp());
+    }
+  }
+
+  return mediapipe::OkStatus();
+}
+
+mediapipe::Status FrameAnnotationToTimedBoxListCalculator::Close(
+    CalculatorContext* cc) {
+  return mediapipe::OkStatus();
+}
+
+}  // namespace mediapipe

mediapipe/modules/objectron/calculators/frame_annotation_tracker.cc

@@ -0,0 +1,102 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/modules/objectron/calculators/frame_annotation_tracker.h"
+
+#include "absl/container/flat_hash_set.h"
+#include "mediapipe/framework/port/logging.h"
+#include "mediapipe/modules/objectron/calculators/annotation_data.pb.h"
+#include "mediapipe/modules/objectron/calculators/box_util.h"
+#include "mediapipe/util/tracking/box_tracker.pb.h"
+
+namespace mediapipe {
+
+void FrameAnnotationTracker::AddDetectionResult(
+    const FrameAnnotation& frame_annotation) {
+  const int64 time_us =
+      static_cast<int64>(std::round(frame_annotation.timestamp()));
+  for (const auto& object_annotation : frame_annotation.annotations()) {
+    detected_objects_[time_us + object_annotation.object_id()] =
+        object_annotation;
+  }
+}
+
+FrameAnnotation FrameAnnotationTracker::ConsolidateTrackingResult(
+    const TimedBoxProtoList& tracked_boxes,
+    absl::flat_hash_set<int>* cancel_object_ids) {
+  CHECK(cancel_object_ids != nullptr);
+  FrameAnnotation frame_annotation;
+  std::vector<int64> keys_to_be_deleted;
+  for (const auto& detected_obj : detected_objects_) {
+    const int object_id = detected_obj.second.object_id();
+    if (cancel_object_ids->contains(object_id)) {
+      // Remember duplicated detections' keys.
+      keys_to_be_deleted.push_back(detected_obj.first);
+      continue;
+    }
+    TimedBoxProto ref_box;
+    for (const auto& box : tracked_boxes.box()) {
+      if (box.id() == object_id) {
+        ref_box = box;
+        break;
+      }
+    }
+    if (!ref_box.has_id() || ref_box.id() < 0) {
+      LOG(ERROR) << "Can't find matching tracked box for object id: "
+                 << object_id << ". Likely lost tracking of it.";
+      keys_to_be_deleted.push_back(detected_obj.first);
+      continue;
+    }
+
+    // Find duplicated boxes
+    for (const auto& box : tracked_boxes.box()) {
+      if (box.id() != object_id) {
+        if (ComputeBoxIoU(ref_box, box) > iou_threshold_) {
+          cancel_object_ids->insert(box.id());
+        }
+      }
+    }
+
+    // Map ObjectAnnotation from detection to tracked time.
+    // First, gather all keypoints from source detection.
+    std::vector<cv::Point2f> key_points;
+    for (const auto& keypoint : detected_obj.second.keypoints()) {
+      key_points.push_back(
+          cv::Point2f(keypoint.point_2d().x(), keypoint.point_2d().y()));
+    }
+    // Second, find source box.
+    TimedBoxProto src_box;
+    ComputeBoundingRect(key_points, &src_box);
+    ObjectAnnotation* tracked_obj = frame_annotation.add_annotations();
+    tracked_obj->set_object_id(ref_box.id());
+    // Finally, map all keypoints in the source detection to tracked location.
+    for (const auto& keypoint : detected_obj.second.keypoints()) {
+      cv::Point2f dst = MapPoint(
+          src_box, ref_box,
+          cv::Point2f(keypoint.point_2d().x(), keypoint.point_2d().y()),
+          img_width_, img_height_);
+      auto* dst_point = tracked_obj->add_keypoints()->mutable_point_2d();
+      dst_point->set_x(dst.x);
+      dst_point->set_y(dst.y);
+    }
+  }
+
+  for (const auto& key : keys_to_be_deleted) {
+    detected_objects_.erase(key);
+  }
+
+  return frame_annotation;
+}
+
+}  // namespace mediapipe

mediapipe/modules/objectron/calculators/frame_annotation_tracker.h

@@ -0,0 +1,62 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_MODULES_OBJECTRON_CALCULATORS_FRAME_ANNOTATION_TRACKER_H_
+#define MEDIAPIPE_MODULES_OBJECTRON_CALCULATORS_FRAME_ANNOTATION_TRACKER_H_
+
+#include <functional>
+
+#include "absl/container/btree_map.h"
+#include "absl/container/flat_hash_set.h"
+#include "mediapipe/framework/port/integral_types.h"
+#include "mediapipe/modules/objectron/calculators/annotation_data.pb.h"
+#include "mediapipe/util/tracking/box_tracker.pb.h"
+
+namespace mediapipe {
+
+class FrameAnnotationTracker {
+ public:
+  // If two bounding boxes have IoU over iou_threshold, then we consider them
+  // describing the same object.
+  FrameAnnotationTracker(float iou_threshold, float img_width, float img_height)
+      : iou_threshold_(iou_threshold),
+        img_width_(img_width),
+        img_height_(img_height) {}
+
+  // Adds detection results from an external detector.
+  void AddDetectionResult(const FrameAnnotation& frame_annotation);
+
+  // Consolidates tracking result from an external tracker, associates with
+  // the detection result by the object id, and produces the corresponding
+  // result in FrameAnnotation. When there are duplicates, output the ids that
+  // need to be cancelled in cancel_object_ids.
+  // Note that the returned FrameAnnotation is missing timestamp. Need to fill
+  // that field.
+  FrameAnnotation ConsolidateTrackingResult(
+      const TimedBoxProtoList& tracked_boxes,
+      absl::flat_hash_set<int>* cancel_object_ids);
+
+ private:
+  float iou_threshold_;
+  float img_width_;
+  float img_height_;
+  // Cached detection results over time.
+  // Key is timestamp_us + object_id.
+  absl::btree_map<int64, ObjectAnnotation, std::greater<int64>>
+      detected_objects_;
+};
+
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_MODULES_OBJECTRON_CALCULATORS_FRAME_ANNOTATION_TRACKER_H_

mediapipe/modules/objectron/calculators/frame_annotation_tracker_calculator.cc

@@ -0,0 +1,137 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "absl/container/flat_hash_set.h"
+#include "absl/memory/memory.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/modules/objectron/calculators/annotation_data.pb.h"
+#include "mediapipe/modules/objectron/calculators/frame_annotation_tracker.h"
+#include "mediapipe/modules/objectron/calculators/frame_annotation_tracker_calculator.pb.h"
+#include "mediapipe/util/tracking/box_tracker.pb.h"
+
+namespace {
+constexpr char kInputFrameAnnotationTag[] = "FRAME_ANNOTATION";
+constexpr char kInputTrackedBoxesTag[] = "TRACKED_BOXES";
+constexpr char kOutputTrackedFrameAnnotationTag[] = "TRACKED_FRAME_ANNOTATION";
+constexpr char kOutputCancelObjectIdTag[] = "CANCEL_OBJECT_ID";
+}  // namespace
+
+namespace mediapipe {
+
+// Tracks frame annotations seeded/updated by FRAME_ANNOTATION input_stream.
+// When using this calculator, make sure FRAME_ANNOTATION and TRACKED_BOXES
+// are in different sync set.
+//
+// Input:
+//  FRAME_ANNOTATION - frame annotation.
+//  TRACKED_BOXES - 2d box tracking result
+// Output:
+//  TRACKED_FRAME_ANNOTATION - annotation inferred from 2d tracking result.
+//  CANCEL_OBJECT_ID - object id that needs to be cancelled from the tracker.
+//
+// Usage example:
+// node {
+//   calculator: "FrameAnnotationTrackerCalculator"
+//   input_stream: "FRAME_ANNOTATION:frame_annotation"
+//   input_stream: "TRACKED_BOXES:tracked_boxes"
+//   output_stream: "TRACKED_FRAME_ANNOTATION:tracked_frame_annotation"
+//   output_stream: "CANCEL_OBJECT_ID:cancel_object_id"
+// }
+class FrameAnnotationTrackerCalculator : public CalculatorBase {
+ public:
+  static mediapipe::Status GetContract(CalculatorContract* cc);
+
+  mediapipe::Status Open(CalculatorContext* cc) override;
+  mediapipe::Status Process(CalculatorContext* cc) override;
+  mediapipe::Status Close(CalculatorContext* cc) override;
+
+ private:
+  std::unique_ptr<FrameAnnotationTracker> frame_annotation_tracker_;
+};
+REGISTER_CALCULATOR(FrameAnnotationTrackerCalculator);
+
+mediapipe::Status FrameAnnotationTrackerCalculator::GetContract(
+    CalculatorContract* cc) {
+  RET_CHECK(!cc->Inputs().GetTags().empty());
+  RET_CHECK(!cc->Outputs().GetTags().empty());
+
+  if (cc->Inputs().HasTag(kInputFrameAnnotationTag)) {
+    cc->Inputs().Tag(kInputFrameAnnotationTag).Set<FrameAnnotation>();
+  }
+  if (cc->Inputs().HasTag(kInputTrackedBoxesTag)) {
+    cc->Inputs().Tag(kInputTrackedBoxesTag).Set<TimedBoxProtoList>();
+  }
+  if (cc->Outputs().HasTag(kOutputTrackedFrameAnnotationTag)) {
+    cc->Outputs().Tag(kOutputTrackedFrameAnnotationTag).Set<FrameAnnotation>();
+  }
+  if (cc->Outputs().HasTag(kOutputCancelObjectIdTag)) {
+    cc->Outputs().Tag(kOutputCancelObjectIdTag).Set<int>();
+  }
+  return mediapipe::OkStatus();
+}
+
+mediapipe::Status FrameAnnotationTrackerCalculator::Open(
+    CalculatorContext* cc) {
+  const auto& options = cc->Options<FrameAnnotationTrackerCalculatorOptions>();
+  frame_annotation_tracker_ = absl::make_unique<FrameAnnotationTracker>(
+      options.iou_threshold(), options.img_width(), options.img_height());
+  return mediapipe::OkStatus();
+}
+
+mediapipe::Status FrameAnnotationTrackerCalculator::Process(
+    CalculatorContext* cc) {
+  if (cc->Inputs().HasTag(kInputFrameAnnotationTag) &&
+      !cc->Inputs().Tag(kInputFrameAnnotationTag).IsEmpty()) {
+    frame_annotation_tracker_->AddDetectionResult(
+        cc->Inputs().Tag(kInputFrameAnnotationTag).Get<FrameAnnotation>());
+  }
+  if (cc->Inputs().HasTag(kInputTrackedBoxesTag) &&
+      !cc->Inputs().Tag(kInputTrackedBoxesTag).IsEmpty() &&
+      cc->Outputs().HasTag(kOutputTrackedFrameAnnotationTag)) {
+    absl::flat_hash_set<int> cancel_object_ids;
+    auto output_frame_annotation = absl::make_unique<FrameAnnotation>();
+    *output_frame_annotation =
+        frame_annotation_tracker_->ConsolidateTrackingResult(
+            cc->Inputs().Tag(kInputTrackedBoxesTag).Get<TimedBoxProtoList>(),
+            &cancel_object_ids);
+    output_frame_annotation->set_timestamp(cc->InputTimestamp().Microseconds());
+
+    cc->Outputs()
+        .Tag(kOutputTrackedFrameAnnotationTag)
+        .Add(output_frame_annotation.release(), cc->InputTimestamp());
+
+    if (cc->Outputs().HasTag(kOutputCancelObjectIdTag)) {
+      auto packet_timestamp = cc->InputTimestamp();
+      for (const auto& id : cancel_object_ids) {
+        // The timestamp is incremented (by 1 us) because currently the box
+        // tracker calculator only accepts one cancel object ID for any given
+        // timestamp.
+        cc->Outputs()
+            .Tag(kOutputCancelObjectIdTag)
+            .AddPacket(mediapipe::MakePacket<int>(id).At(packet_timestamp++));
+      }
+    }
+  }
+
+  return mediapipe::OkStatus();
+}
+
+mediapipe::Status FrameAnnotationTrackerCalculator::Close(
+    CalculatorContext* cc) {
+  return mediapipe::OkStatus();
+}
+
+}  // namespace mediapipe

mediapipe/modules/objectron/calculators/frame_annotation_tracker_calculator.proto

@@ -0,0 +1,36 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// The option proto for the FrameAnnotationTrackerCalculatorOptions.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+
+message FrameAnnotationTrackerCalculatorOptions {
+  extend CalculatorOptions {
+    optional FrameAnnotationTrackerCalculatorOptions ext = 291291253;
+  }
+
+  // The threshold on intersection-over-union (IoU). We consider
+  // boxes with IoU larger than this threshold to be the duplicates.
+  optional float iou_threshold = 1 [default = 0.5];
+
+  // We need image dimension to properly compute annotation locations.
+  optional float img_width = 2;
+
+  optional float img_height = 3;
+}

mediapipe/modules/objectron/calculators/frame_annotation_tracker_test.cc

@@ -0,0 +1,143 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/modules/objectron/calculators/frame_annotation_tracker.h"
+
+#include "absl/container/flat_hash_set.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/logging.h"
+#include "mediapipe/modules/objectron/calculators/annotation_data.pb.h"
+#include "mediapipe/util/tracking/box_tracker.pb.h"
+
+namespace mediapipe {
+namespace {
+
+// Create a new object annotation by shifting a reference
+// object annotation.
+ObjectAnnotation ShiftObject2d(const ObjectAnnotation& ref_obj, float dx,
+                               float dy) {
+  ObjectAnnotation obj = ref_obj;
+  for (auto& keypoint : *(obj.mutable_keypoints())) {
+    const float ref_x = keypoint.point_2d().x();
+    const float ref_y = keypoint.point_2d().y();
+    keypoint.mutable_point_2d()->set_x(ref_x + dx);
+    keypoint.mutable_point_2d()->set_y(ref_y + dy);
+  }
+  return obj;
+}
+
+TimedBoxProto ShiftBox(const TimedBoxProto& ref_box, float dx, float dy) {
+  TimedBoxProto box = ref_box;
+  box.set_top(ref_box.top() + dy);
+  box.set_bottom(ref_box.bottom() + dy);
+  box.set_left(ref_box.left() + dx);
+  box.set_right(ref_box.right() + dx);
+  return box;
+}
+
+// Constructs a fixed ObjectAnnotation.
+ObjectAnnotation ConstructFixedObject(
+    const std::vector<std::vector<float>>& points) {
+  ObjectAnnotation obj;
+  for (const auto& point : points) {
+    auto* keypoint = obj.add_keypoints();
+    CHECK_EQ(2, point.size());
+    keypoint->mutable_point_2d()->set_x(point[0]);
+    keypoint->mutable_point_2d()->set_y(point[1]);
+  }
+  return obj;
+}
+
+TEST(FrameAnnotationTrackerTest, TestConsolidation) {
+  // Add 4 detections represented by FrameAnnotation, of which 3 correspond
+  // to the same object.
+  ObjectAnnotation object1, object2, object3, object4;
+  // The bounding rectangle for these object keypoints is:
+  // x: [0.2, 0.5], y: [0.1, 0.4]
+  object3 = ConstructFixedObject({{0.35f, 0.25f},
+                                  {0.3f, 0.3f},
+                                  {0.2f, 0.4f},
+                                  {0.3f, 0.1f},
+                                  {0.2f, 0.2f},
+                                  {0.5f, 0.3f},
+                                  {0.4f, 0.4f},
+                                  {0.5f, 0.1f},
+                                  {0.4f, 0.2f}});
+  object3.set_object_id(3);
+  object1 = ShiftObject2d(object3, -0.05f, -0.05f);
+  object1.set_object_id(1);
+  object2 = ShiftObject2d(object3, 0.05f, 0.05f);
+  object2.set_object_id(2);
+  object4 = ShiftObject2d(object3, 0.2f, 0.2f);
+  object4.set_object_id(4);
+  FrameAnnotation frame_annotation_1;
+  frame_annotation_1.set_timestamp(30 * 1000);  // 30ms
+  *(frame_annotation_1.add_annotations()) = object1;
+  *(frame_annotation_1.add_annotations()) = object4;
+  FrameAnnotation frame_annotation_2;
+  frame_annotation_2.set_timestamp(60 * 1000);  // 60ms
+  *(frame_annotation_2.add_annotations()) = object2;
+  FrameAnnotation frame_annotation_3;
+  frame_annotation_3.set_timestamp(90 * 1000);  // 90ms
+  *(frame_annotation_3.add_annotations()) = object3;
+
+  FrameAnnotationTracker frame_annotation_tracker(/*iou_threshold*/ 0.5f, 1.0f,
+                                                  1.0f);
+  frame_annotation_tracker.AddDetectionResult(frame_annotation_1);
+  frame_annotation_tracker.AddDetectionResult(frame_annotation_2);
+  frame_annotation_tracker.AddDetectionResult(frame_annotation_3);
+
+  TimedBoxProtoList timed_box_proto_list;
+  TimedBoxProto* timed_box_proto = timed_box_proto_list.add_box();
+  timed_box_proto->set_top(0.4f);
+  timed_box_proto->set_bottom(0.7f);
+  timed_box_proto->set_left(0.6f);
+  timed_box_proto->set_right(0.9f);
+  timed_box_proto->set_id(3);
+  timed_box_proto->set_time_msec(150);
+  timed_box_proto = timed_box_proto_list.add_box();
+  *timed_box_proto = ShiftBox(timed_box_proto_list.box(0), 0.01f, 0.01f);
+  timed_box_proto->set_id(1);
+  timed_box_proto->set_time_msec(150);
+  timed_box_proto = timed_box_proto_list.add_box();
+  *timed_box_proto = ShiftBox(timed_box_proto_list.box(0), -0.01f, -0.01f);
+  timed_box_proto->set_id(2);
+  timed_box_proto->set_time_msec(150);
+  absl::flat_hash_set<int> cancel_object_ids;
+  FrameAnnotation tracked_detection =
+      frame_annotation_tracker.ConsolidateTrackingResult(timed_box_proto_list,
+                                                         &cancel_object_ids);
+  EXPECT_EQ(2, cancel_object_ids.size());
+  EXPECT_EQ(1, cancel_object_ids.count(1));
+  EXPECT_EQ(1, cancel_object_ids.count(2));
+  EXPECT_EQ(1, tracked_detection.annotations_size());
+  EXPECT_EQ(3, tracked_detection.annotations(0).object_id());
+  EXPECT_EQ(object3.keypoints_size(),
+            tracked_detection.annotations(0).keypoints_size());
+  const float x_offset = 0.4f;
+  const float y_offset = 0.3f;
+  const float tolerance = 1e-5f;
+  for (int i = 0; i < object3.keypoints_size(); ++i) {
+    const auto& point_2d =
+        tracked_detection.annotations(0).keypoints(i).point_2d();
+    EXPECT_NEAR(point_2d.x(), object3.keypoints(i).point_2d().x() + x_offset,
+                tolerance);
+    EXPECT_NEAR(point_2d.y(), object3.keypoints(i).point_2d().y() + y_offset,
+                tolerance);
+  }
+}
+
+}  // namespace
+}  // namespace mediapipe

mediapipe/modules/objectron/calculators/landmarks_to_frame_annotation_calculator.cc

@@ -0,0 +1,76 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+
+#include "absl/memory/memory.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/landmark.pb.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/modules/objectron/calculators/annotation_data.pb.h"
+
+namespace mediapipe {
+
+namespace {
+
+constexpr char kInputLandmarksTag[] = "LANDMARKS";
+constexpr char kOutputFrameAnnotationTag[] = "FRAME_ANNOTATION";
+
+}  // namespace
+
+// A calculator that converts NormalizedLandmarkList to FrameAnnotation proto.
+class LandmarksToFrameAnnotationCalculator : public CalculatorBase {
+ public:
+  static ::mediapipe::Status GetContract(CalculatorContract* cc);
+  ::mediapipe::Status Process(CalculatorContext* cc) override;
+};
+REGISTER_CALCULATOR(LandmarksToFrameAnnotationCalculator);
+
+::mediapipe::Status LandmarksToFrameAnnotationCalculator::GetContract(
+    CalculatorContract* cc) {
+  RET_CHECK(!cc->Inputs().GetTags().empty());
+  RET_CHECK(!cc->Outputs().GetTags().empty());
+
+  if (cc->Inputs().HasTag(kInputLandmarksTag)) {
+    cc->Inputs().Tag(kInputLandmarksTag).Set<NormalizedLandmarkList>();
+  }
+
+  if (cc->Outputs().HasTag(kOutputFrameAnnotationTag)) {
+    cc->Outputs().Tag(kOutputFrameAnnotationTag).Set<FrameAnnotation>();
+  }
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status LandmarksToFrameAnnotationCalculator::Process(
+    CalculatorContext* cc) {
+  auto frame_annotation = absl::make_unique<FrameAnnotation>();
+  auto* box_annotation = frame_annotation->add_annotations();
+
+  const auto& landmarks =
+      cc->Inputs().Tag(kInputLandmarksTag).Get<NormalizedLandmarkList>();
+  RET_CHECK_GT(landmarks.landmark_size(), 0)
+      << "Input landmark vector is empty.";
+  for (int i = 0; i < landmarks.landmark_size(); ++i) {
+    auto* point2d = box_annotation->add_keypoints()->mutable_point_2d();
+    point2d->set_x(landmarks.landmark(i).x());
+    point2d->set_y(landmarks.landmark(i).y());
+  }
+  // Output
+  if (cc->Outputs().HasTag(kOutputFrameAnnotationTag)) {
+    cc->Outputs()
+        .Tag(kOutputFrameAnnotationTag)
+        .Add(frame_annotation.release(), cc->InputTimestamp());
+  }
+  return ::mediapipe::OkStatus();
+}
+
+}  // namespace mediapipe

mediapipe/modules/objectron/calculators/lift_2d_frame_annotation_to_3d_calculator.cc

@@ -0,0 +1,168 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <memory>
+#include <unordered_map>
+#include <vector>
+
+#include "Eigen/Dense"
+#include "absl/memory/memory.h"
+#include "absl/strings/str_format.h"
+#include "absl/types/span.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/deps/file_path.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/modules/objectron/calculators/annotation_data.pb.h"
+#include "mediapipe/modules/objectron/calculators/decoder.h"
+#include "mediapipe/modules/objectron/calculators/lift_2d_frame_annotation_to_3d_calculator.pb.h"
+#include "mediapipe/modules/objectron/calculators/tensor_util.h"
+
+namespace {
+constexpr char kInputStreamTag[] = "FRAME_ANNOTATION";
+constexpr char kOutputStreamTag[] = "LIFTED_FRAME_ANNOTATION";
+
+// Each detection object will be assigned an unique id that starts from 1.
+static int object_id = 0;
+
+inline int GetNextObjectId() { return ++object_id; }
+}  // namespace
+
+namespace mediapipe {
+
+// Lifted the 2D points in a tracked frame annotation to 3D.
+//
+// Input:
+//  FRAME_ANNOTATIONS - Frame annotations with detected 2D points
+// Output:
+//  LIFTED_FRAME_ANNOTATIONS - Result FrameAnnotation with lifted 3D points.
+//
+// Usage example:
+// node {
+//   calculator: "Lift2DFrameAnnotationTo3DCalculator"
+//   input_stream: "FRAME_ANNOTATIONS:tracked_annotations"
+//   output_stream: "LIFTED_FRAME_ANNOTATIONS:lifted_3d_annotations"
+// }
+class Lift2DFrameAnnotationTo3DCalculator : public CalculatorBase {
+ public:
+  static ::mediapipe::Status GetContract(CalculatorContract* cc);
+
+  ::mediapipe::Status Open(CalculatorContext* cc) override;
+  ::mediapipe::Status Process(CalculatorContext* cc) override;
+  ::mediapipe::Status Close(CalculatorContext* cc) override;
+
+ private:
+  ::mediapipe::Status ProcessCPU(CalculatorContext* cc,
+                                 FrameAnnotation* output_objects);
+  ::mediapipe::Status LoadOptions(CalculatorContext* cc);
+
+  // Increment and assign object ID for each detected object.
+  // In a single MediaPipe session, the IDs are unique.
+  // Also assign timestamp for the FrameAnnotation to be the input packet
+  // timestamp.
+  void AssignObjectIdAndTimestamp(int64 timestamp_us,
+                                  FrameAnnotation* annotation);
+  std::unique_ptr<Decoder> decoder_;
+  ::mediapipe::Lift2DFrameAnnotationTo3DCalculatorOptions options_;
+  Eigen::Matrix<float, 4, 4, Eigen::RowMajor> projection_matrix_;
+};
+REGISTER_CALCULATOR(Lift2DFrameAnnotationTo3DCalculator);
+
+::mediapipe::Status Lift2DFrameAnnotationTo3DCalculator::GetContract(
+    CalculatorContract* cc) {
+  RET_CHECK(cc->Inputs().HasTag(kInputStreamTag));
+  RET_CHECK(cc->Outputs().HasTag(kOutputStreamTag));
+  cc->Inputs().Tag(kInputStreamTag).Set<FrameAnnotation>();
+  cc->Outputs().Tag(kOutputStreamTag).Set<FrameAnnotation>();
+
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status Lift2DFrameAnnotationTo3DCalculator::Open(
+    CalculatorContext* cc) {
+  MP_RETURN_IF_ERROR(LoadOptions(cc));
+  // clang-format off
+  projection_matrix_ <<
+      1.5731,   0,       0,    0,
+      0,   2.0975,       0,    0,
+      0,        0, -1.0002, -0.2,
+      0,        0,      -1,    0;
+  // clang-format on
+
+  decoder_ = absl::make_unique<Decoder>(
+      BeliefDecoderConfig(options_.decoder_config()));
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status Lift2DFrameAnnotationTo3DCalculator::Process(
+    CalculatorContext* cc) {
+  if (cc->Inputs().Tag(kInputStreamTag).IsEmpty()) {
+    return ::mediapipe::OkStatus();
+  }
+
+  auto output_objects = absl::make_unique<FrameAnnotation>();
+
+  MP_RETURN_IF_ERROR(ProcessCPU(cc, output_objects.get()));
+
+  // Output
+  if (cc->Outputs().HasTag(kOutputStreamTag)) {
+    cc->Outputs()
+        .Tag(kOutputStreamTag)
+        .Add(output_objects.release(), cc->InputTimestamp());
+  }
+
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status Lift2DFrameAnnotationTo3DCalculator::ProcessCPU(
+    CalculatorContext* cc, FrameAnnotation* output_objects) {
+  const auto& input_frame_annotations =
+      cc->Inputs().Tag(kInputStreamTag).Get<FrameAnnotation>();
+  // Copy the input frame annotation to the output
+  *output_objects = input_frame_annotations;
+
+  auto status = decoder_->Lift2DTo3D(projection_matrix_, /*portrait*/ true,
+                                     output_objects);
+  if (!status.ok()) {
+    LOG(ERROR) << status;
+    return status;
+  }
+  AssignObjectIdAndTimestamp(cc->InputTimestamp().Microseconds(),
+                             output_objects);
+
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status Lift2DFrameAnnotationTo3DCalculator::Close(
+    CalculatorContext* cc) {
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status Lift2DFrameAnnotationTo3DCalculator::LoadOptions(
+    CalculatorContext* cc) {
+  // Get calculator options specified in the graph.
+  options_ =
+      cc->Options<::mediapipe::Lift2DFrameAnnotationTo3DCalculatorOptions>();
+
+  return ::mediapipe::OkStatus();
+}
+
+void Lift2DFrameAnnotationTo3DCalculator::AssignObjectIdAndTimestamp(
+    int64 timestamp_us, FrameAnnotation* annotation) {
+  for (auto& ann : *annotation->mutable_annotations()) {
+    ann.set_object_id(GetNextObjectId());
+  }
+  annotation->set_timestamp(timestamp_us);
+}
+
+}  // namespace mediapipe

mediapipe/modules/objectron/calculators/lift_2d_frame_annotation_to_3d_calculator.proto

@@ -0,0 +1,30 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// The option proto for the Lift2DFrameAnnotationTo3DCalculatorOptions.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+import "mediapipe/modules/objectron/calculators/belief_decoder_config.proto";
+
+message Lift2DFrameAnnotationTo3DCalculatorOptions {
+  extend CalculatorOptions {
+    optional Lift2DFrameAnnotationTo3DCalculatorOptions ext = 290166284;
+  }
+
+  optional BeliefDecoderConfig decoder_config = 1;
+}

mediapipe/modules/objectron/calculators/model.cc

@@ -0,0 +1,101 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/modules/objectron/calculators/model.h"
+
+#include "mediapipe/framework/port/logging.h"
+
+namespace mediapipe {
+
+void Model::SetTransformation(const Eigen::Matrix4f& transform) {
+  transformation_ = transform;
+}
+
+void Model::SetTranslation(const Eigen::Vector3f& translation) {
+  transformation_.col(3).template head<3>() = translation;
+}
+
+void Model::SetRotation(float roll, float pitch, float yaw) {
+  // In our coordinate system, Y is up. We first rotate the object around Y
+  // (yaw), then around Z (pitch), and finally around X (roll).
+  Eigen::Matrix3f r;
+  r = Eigen::AngleAxisf(yaw, Eigen::Vector3f::UnitY()) *
+      Eigen::AngleAxisf(pitch, Eigen::Vector3f::UnitZ()) *
+      Eigen::AngleAxisf(roll, Eigen::Vector3f::UnitX());
+  transformation_.topLeftCorner<3, 3>() = r;
+}
+
+void Model::SetRotation(const Eigen::Matrix3f& rotation) {
+  transformation_.topLeftCorner<3, 3>() = rotation;
+}
+
+void Model::SetScale(const Eigen::Vector3f& scale) { scale_ = scale; }
+
+void Model::SetCategory(const std::string& category) { category_ = category; }
+
+const Eigen::Vector3f Model::GetRotationAngles() const {
+  Vector3f ypr = transformation_.topLeftCorner<3, 3>().eulerAngles(1, 2, 0);
+  return Vector3f(ypr(2), ypr(1), ypr(0));  // swap YPR with RPY
+}
+
+const Eigen::Matrix4f& Model::GetTransformation() const {
+  return transformation_;
+}
+
+const Eigen::Vector3f& Model::GetScale() const { return scale_; }
+
+const Eigen::Ref<const Eigen::Vector3f> Model::GetTranslation() const {
+  return transformation_.col(3).template head<3>();
+}
+
+const Eigen::Ref<const Eigen::Matrix3f> Model::GetRotation() const {
+  return transformation_.template topLeftCorner<3, 3>();
+}
+
+const std::string& Model::GetCategory() const { return category_; }
+
+void Model::Deserialize(const Object& obj) {
+  CHECK_EQ(obj.rotation_size(), 9);
+  CHECK_EQ(obj.translation_size(), 3);
+  CHECK_EQ(obj.scale_size(), 3);
+  category_ = obj.category();
+
+  using RotationMatrix = Eigen::Matrix<float, 3, 3, Eigen::RowMajor>;
+  transformation_.setIdentity();
+  transformation_.topLeftCorner<3, 3>() =
+      Eigen::Map<const RotationMatrix>(obj.rotation().data());
+  transformation_.col(3).head<3>() =
+      Eigen::Map<const Eigen::Vector3f>(obj.translation().data());
+  scale_ = Eigen::Map<const Eigen::Vector3f>(obj.scale().data());
+  Update();
+}
+
+void Model::Serialize(Object* obj) {
+  obj->set_category(category_);
+  for (int i = 0; i < 3; ++i) {
+    for (int j = 0; j < 3; ++j) {
+      obj->add_rotation(transformation_(i, j));
+    }
+  }
+
+  for (int i = 0; i < 3; ++i) {
+    obj->add_translation(transformation_(i, 3));
+  }
+
+  for (int i = 0; i < 3; ++i) {
+    obj->add_scale(scale_[i]);
+  }
+}
+
+}  // namespace mediapipe

mediapipe/modules/objectron/calculators/model.h

@@ -0,0 +1,92 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_MODULES_OBJECTRON_CALCULATORS_MODEL_H_
+#define MEDIAPIPE_MODULES_OBJECTRON_CALCULATORS_MODEL_H_
+
+#include "mediapipe/modules/objectron/calculators/annotation_data.pb.h"
+#include "mediapipe/modules/objectron/calculators/object.pb.h"
+#include "mediapipe/modules/objectron/calculators/types.h"
+
+namespace mediapipe {
+
+class Model {
+ public:
+  EIGEN_MAKE_ALIGNED_OPERATOR_NEW
+
+  enum Type {
+    kVisualizationOnly = 0,
+    kBoundingBox,
+    kSkeleton,
+    kShape,  // Shape is a virtual object.
+    kNumModes,
+  };
+
+  virtual ~Model() = default;
+
+  virtual void SetTransformation(const Eigen::Matrix4f& transform);
+  virtual void SetTranslation(const Eigen::Vector3f& translation);
+
+  // Compute the rotation matrix from these angles and update the transformation
+  // matrix accordingly
+  virtual void SetRotation(float roll, float pitch, float yaw);
+  virtual void SetRotation(const Eigen::Matrix3f& rotation);
+  virtual void SetScale(const Eigen::Vector3f& scale);
+  virtual void SetCategory(const std::string& category);
+  virtual size_t GetNumberKeypoints() const { return number_keypoints_; }
+
+  // Gets Euler angles in the order of roll, pitch, yaw.
+  virtual const Eigen::Vector3f GetRotationAngles() const;
+  virtual const Eigen::Matrix4f& GetTransformation() const;
+  virtual const Eigen::Vector3f& GetScale() const;
+  virtual const Eigen::Ref<const Eigen::Vector3f> GetTranslation() const;
+  virtual const Eigen::Ref<const Eigen::Matrix3f> GetRotation() const;
+  virtual const std::string& GetCategory() const;
+
+  // Update the model's keypoints in the world-coordinate system.
+  // The update includes transforming the model to the world-coordinate system
+  // as well as scaling the model.
+  // The user is expected to call this function after Setting the rotation,
+  // orientation or the scale of the model to get an updated model.
+  virtual void Update() = 0;
+
+  // Update the model's parameters (orientation, position, and scale) from the
+  // user-provided variables.
+  virtual void Adjust(const std::vector<float>& variables) = 0;
+
+  // Returns a pointer to the model's keypoints.
+  // Use Eigen::Map to cast the pointer back to Vector3 or Vector4
+  virtual const float* GetVertex(size_t id) const = 0;
+  virtual float* GetVertex(size_t id) = 0;
+  virtual void Deserialize(const Object& obj);
+  virtual void Serialize(Object* obj);
+
+  // TODO: make member variables protected, and add public apis.
+  // 4x4 transformation matrix mapping the first keypoint to world coordinate
+  Eigen::Matrix4f transformation_;
+  Eigen::Vector3f scale_;  // width, height, depth
+  Type model_type_;
+  size_t number_keypoints_;
+  std::string category_;
+
+ protected:
+  Model(Type type, size_t number_keypoints, const std::string& category)
+      : model_type_(type),
+        number_keypoints_(number_keypoints),
+        category_(category) {}
+};
+
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_MODULES_OBJECTRON_CALCULATORS_MODEL_H_

mediapipe/modules/objectron/calculators/object.proto

@@ -0,0 +1,124 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto3";
+
+package mediapipe;
+
+message KeyPoint {
+  // The position of the keypoint in the local coordinate system of the rigid
+  // object.
+  float x = 1;
+  float y = 2;
+  float z = 3;
+
+  // Sphere around the keypoint, indiciating annotator's confidence of the
+  // position in meters.
+  float confidence_radius = 4;
+
+  // The name of the keypoint (e.g. legs, head, etc.).
+  // Does not have to be unique.
+  string name = 5;
+
+  // Indicates whether the keypoint is hidden or not.
+  bool hidden = 6;
+}
+
+message Object {
+  // Unique object id through a sequence. There might be multiple objects of
+  // the same label in this sequence.
+  int32 id = 1;
+
+  // Describes what category an object is. E.g. object class, attribute,
+  // instance or person identity. This provides additional context for the
+  // object type.
+  string category = 2;
+
+  enum Type {
+    UNDEFINED_TYPE = 0;
+    BOUNDING_BOX = 1;
+    SKELETON = 2;
+  }
+
+  Type type = 3;
+
+  // 3x3 row-major rotation matrix describing the orientation of the rigid
+  // object's frame of reference in the world-coordinate system.
+  repeated float rotation = 4;
+
+  // 3x1 vector describing the translation of the rigid object's frame of
+  // reference in the world-coordinate system in meters.
+  repeated float translation = 5;
+
+  // 3x1 vector describing the scale of the rigid object's frame of reference in
+  // the world-coordinate system in meters.
+  repeated float scale = 6;
+
+  // List of all the key points associated with this object in the object
+  // coordinate system.
+  // The first keypoint is always the object's frame of reference,
+  // e.g. the centroid of the box.
+  // E.g. bounding box with its center as frame of reference, the 9 keypoints :
+  // {0., 0., 0.},
+  // {-.5, -.5, -.5}, {-.5, -.5, +.5}, {-.5, +.5, -.5}, {-.5, +.5, +.5},
+  // {+.5, -.5, -.5}, {+.5, -.5, +.5}, {+.5, +.5, -.5}, {+.5, +.5, +.5}
+  // To get the bounding box in the world-coordinate system, we first scale the
+  // box then transform the scaled box.
+  // For example, bounding box in the world coordinate system is
+  // rotation * scale * keypoints + translation
+  repeated KeyPoint keypoints = 7;
+
+  // Enum to reflect how this object is created.
+  enum Method {
+    UNKNOWN_METHOD = 0;
+    ANNOTATION = 1;    // Created by data annotation.
+    AUGMENTATION = 2;  // Created by data augmentation.
+  }
+  Method method = 8;
+}
+
+// The edge connecting two keypoints together
+message Edge {
+  // keypoint id of the edge's source
+  int32 source = 1;
+
+  // keypoint id of the edge's sink
+  int32 sink = 2;
+}
+
+// The skeleton template for different objects (e.g. humans, chairs, hands, etc)
+// The annotation tool reads the skeleton template dictionary.
+message Skeleton {
+  // The origin keypoint in the object coordinate system. (i.e. Point 0, 0, 0)
+  int32 reference_keypoint = 1;
+
+  // The skeleton's category (e.g. human, chair, hand.). Should be unique in the
+  // dictionary.
+  string category = 2;
+
+  // Initialization value for all the keypoints in the skeleton in the object's
+  // local coordinate system. Pursuit will transform these points using object's
+  // transformation to get the keypoint in the world-cooridnate.
+  repeated KeyPoint keypoints = 3;
+
+  // List of edges connecting keypoints
+  repeated Edge edges = 4;
+}
+
+// The list of all the modeled skeletons in our library. These models can be
+// objects (chairs, desks, etc), humans (full pose, hands, faces, etc), or box.
+// We can have multiple skeletons in the same file.
+message Skeletons {
+  repeated Skeleton object = 1;
+}

mediapipe/modules/objectron/calculators/tensor_util.cc

@@ -0,0 +1,48 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/modules/objectron/calculators/tensor_util.h"
+
+#include "mediapipe/framework/port/logging.h"
+
+namespace mediapipe {
+
+cv::Mat ConvertTfliteTensorToCvMat(const TfLiteTensor& tensor) {
+  // Check tensor is BxCxWxH (size = 4) and the batch size is one(data[0] = 1)
+  CHECK(tensor.dims->size == 4 && tensor.dims->data[0] == 1);
+  CHECK_EQ(kTfLiteFloat32, tensor.type) << "tflite_tensor type is not float";
+
+  const size_t num_output_channels = tensor.dims->data[3];
+  const int dims = 2;
+  const int sizes[] = {tensor.dims->data[1], tensor.dims->data[2]};
+  const int type = CV_MAKETYPE(CV_32F, num_output_channels);
+  return cv::Mat(dims, sizes, type, reinterpret_cast<void*>(tensor.data.f));
+}
+
+cv::Mat ConvertTensorToCvMat(const mediapipe::Tensor& tensor) {
+  // Check tensor is BxCxWxH (size = 4) and the batch size is one(data[0] = 1)
+  CHECK(tensor.shape().dims.size() == 4 && tensor.shape().dims[0] == 1);
+  CHECK_EQ(mediapipe::Tensor::ElementType::kFloat32 == tensor.element_type(),
+           true)
+      << "tensor type is not float";
+
+  const size_t num_output_channels = tensor.shape().dims[3];
+  const int dims = 2;
+  const int sizes[] = {tensor.shape().dims[1], tensor.shape().dims[2]};
+  const int type = CV_MAKETYPE(CV_32F, num_output_channels);
+  auto cpu_view = tensor.GetCpuReadView();
+  return cv::Mat(dims, sizes, type, const_cast<void*>(cpu_view.buffer<void>()));
+}
+
+}  // namespace mediapipe

mediapipe/modules/objectron/calculators/tensor_util.h

@@ -0,0 +1,31 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_MODULES_OBJECTRON_CALCULATORS_TENSOR_UTIL_H_
+#define MEDIAPIPE_MODULES_OBJECTRON_CALCULATORS_TENSOR_UTIL_H_
+
+#include "mediapipe/framework/formats/tensor.h"
+#include "mediapipe/framework/port/opencv_core_inc.h"
+#include "tensorflow/lite/interpreter.h"
+
+namespace mediapipe {
+
+// Converts a single channel tflite tensor to a grayscale image
+cv::Mat ConvertTfliteTensorToCvMat(const TfLiteTensor& tensor);
+
+// Converts a single channel tensor to grayscale image
+cv::Mat ConvertTensorToCvMat(const mediapipe::Tensor& tensor);
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_MODULES_OBJECTRON_CALCULATORS_TENSOR_UTIL_H_

mediapipe/modules/objectron/calculators/tensors_to_objects_calculator.cc

@@ -0,0 +1,211 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <memory>
+#include <unordered_map>
+#include <vector>
+
+#include "Eigen/Dense"
+#include "absl/memory/memory.h"
+#include "absl/strings/str_format.h"
+#include "absl/types/span.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/deps/file_path.h"
+#include "mediapipe/framework/formats/tensor.h"
+#include "mediapipe/framework/port/opencv_core_inc.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/modules/objectron/calculators/annotation_data.pb.h"
+#include "mediapipe/modules/objectron/calculators/belief_decoder_config.pb.h"
+#include "mediapipe/modules/objectron/calculators/decoder.h"
+#include "mediapipe/modules/objectron/calculators/tensor_util.h"
+#include "mediapipe/modules/objectron/calculators/tensors_to_objects_calculator.pb.h"
+
+namespace {
+constexpr char kInputStreamTag[] = "TENSORS";
+constexpr char kOutputStreamTag[] = "ANNOTATIONS";
+
+// Each detection object will be assigned an unique id that starts from 1.
+static int object_id = 0;
+
+inline int GetNextObjectId() { return ++object_id; }
+}  // namespace
+
+namespace mediapipe {
+
+// Convert result Tensors from deep pursuit 3d model into FrameAnnotation.
+//
+// Input:
+//  TENSORS - Vector of Tensor of type kFloat32.
+// Output:
+//  ANNOTATIONS - Result FrameAnnotation.
+//
+// Usage example:
+// node {
+//   calculator: "TensorsToObjectsCalculator"
+//   input_stream: "TENSORS:tensors"
+//   output_stream: "ANNOTATIONS:annotations"
+// }
+class TensorsToObjectsCalculator : public CalculatorBase {
+ public:
+  static mediapipe::Status GetContract(CalculatorContract* cc);
+
+  mediapipe::Status Open(CalculatorContext* cc) override;
+  mediapipe::Status Process(CalculatorContext* cc) override;
+  mediapipe::Status Close(CalculatorContext* cc) override;
+
+ private:
+  mediapipe::Status ProcessCPU(CalculatorContext* cc,
+                               FrameAnnotation* output_objects);
+  mediapipe::Status LoadOptions(CalculatorContext* cc);
+  // Takes point_3d in FrameAnnotation, projects to 2D, and overwrite the
+  // point_2d field with the projection.
+  void Project3DTo2D(bool portrait, FrameAnnotation* annotation) const;
+  // Increment and assign object ID for each detected object.
+  // In a single MediaPipe session, the IDs are unique.
+  // Also assign timestamp for the FrameAnnotation to be the input packet
+  // timestamp.
+  void AssignObjectIdAndTimestamp(int64 timestamp_us,
+                                  FrameAnnotation* annotation);
+
+  int num_classes_ = 0;
+  int num_keypoints_ = 0;
+
+  ::mediapipe::TensorsToObjectsCalculatorOptions options_;
+  std::unique_ptr<Decoder> decoder_;
+  Eigen::Matrix<float, 4, 4, Eigen::RowMajor> projection_matrix_;
+};
+REGISTER_CALCULATOR(TensorsToObjectsCalculator);
+
+mediapipe::Status TensorsToObjectsCalculator::GetContract(
+    CalculatorContract* cc) {
+  RET_CHECK(!cc->Inputs().GetTags().empty());
+  RET_CHECK(!cc->Outputs().GetTags().empty());
+
+  if (cc->Inputs().HasTag(kInputStreamTag)) {
+    cc->Inputs().Tag(kInputStreamTag).Set<std::vector<mediapipe::Tensor>>();
+  }
+
+  if (cc->Outputs().HasTag(kOutputStreamTag)) {
+    cc->Outputs().Tag(kOutputStreamTag).Set<FrameAnnotation>();
+  }
+  return mediapipe::OkStatus();
+}
+
+mediapipe::Status TensorsToObjectsCalculator::Open(CalculatorContext* cc) {
+  MP_RETURN_IF_ERROR(LoadOptions(cc));
+  // clang-format off
+  projection_matrix_ <<
+      1.5731,   0,       0, 0,
+      0,   2.0975,       0, 0,
+      0,        0, -1.0002, -0.2,
+      0,        0,      -1, 0;
+  // clang-format on
+  decoder_ = absl::make_unique<Decoder>(
+      BeliefDecoderConfig(options_.decoder_config()));
+
+  return mediapipe::OkStatus();
+}
+
+mediapipe::Status TensorsToObjectsCalculator::Process(CalculatorContext* cc) {
+  if (cc->Inputs().Tag(kInputStreamTag).IsEmpty()) {
+    return mediapipe::OkStatus();
+  }
+
+  auto output_objects = absl::make_unique<FrameAnnotation>();
+
+  MP_RETURN_IF_ERROR(ProcessCPU(cc, output_objects.get()));
+
+  // Output
+  if (cc->Outputs().HasTag(kOutputStreamTag)) {
+    cc->Outputs()
+        .Tag(kOutputStreamTag)
+        .Add(output_objects.release(), cc->InputTimestamp());
+  }
+
+  return mediapipe::OkStatus();
+}
+
+mediapipe::Status TensorsToObjectsCalculator::ProcessCPU(
+    CalculatorContext* cc, FrameAnnotation* output_objects) {
+  const auto& input_tensors =
+      cc->Inputs().Tag(kInputStreamTag).Get<std::vector<mediapipe::Tensor>>();
+
+  cv::Mat prediction_heatmap = ConvertTensorToCvMat(input_tensors[0]);
+  cv::Mat offsetmap = ConvertTensorToCvMat(input_tensors[1]);
+
+  *output_objects =
+      decoder_->DecodeBoundingBoxKeypoints(prediction_heatmap, offsetmap);
+  auto status = decoder_->Lift2DTo3D(projection_matrix_, /*portrait*/ true,
+                                     output_objects);
+  if (!status.ok()) {
+    LOG(ERROR) << status;
+    return status;
+  }
+  Project3DTo2D(/*portrait*/ true, output_objects);
+  AssignObjectIdAndTimestamp(cc->InputTimestamp().Microseconds(),
+                             output_objects);
+
+  return mediapipe::OkStatus();
+}
+
+mediapipe::Status TensorsToObjectsCalculator::Close(CalculatorContext* cc) {
+  return mediapipe::OkStatus();
+}
+
+mediapipe::Status TensorsToObjectsCalculator::LoadOptions(
+    CalculatorContext* cc) {
+  // Get calculator options specified in the graph.
+  options_ = cc->Options<::mediapipe::TensorsToObjectsCalculatorOptions>();
+
+  num_classes_ = options_.num_classes();
+  num_keypoints_ = options_.num_keypoints();
+
+  // Currently only support 2D when num_values_per_keypoint equals to 2.
+  CHECK_EQ(options_.num_values_per_keypoint(), 2);
+
+  return mediapipe::OkStatus();
+}
+
+void TensorsToObjectsCalculator::Project3DTo2D(
+    bool portrait, FrameAnnotation* annotation) const {
+  for (auto& ann : *annotation->mutable_annotations()) {
+    for (auto& key_point : *ann.mutable_keypoints()) {
+      Eigen::Vector4f point3d;
+      point3d << key_point.point_3d().x(), key_point.point_3d().y(),
+          key_point.point_3d().z(), 1.0f;
+      Eigen::Vector4f point3d_projection = projection_matrix_ * point3d;
+      float u, v;
+      const float inv_w = 1.0f / point3d_projection(3);
+      if (portrait) {
+        u = (point3d_projection(1) * inv_w + 1.0f) * 0.5f;
+        v = (point3d_projection(0) * inv_w + 1.0f) * 0.5f;
+      } else {
+        u = (point3d_projection(0) * inv_w + 1.0f) * 0.5f;
+        v = (1.0f - point3d_projection(1) * inv_w) * 0.5f;
+      }
+      key_point.mutable_point_2d()->set_x(u);
+      key_point.mutable_point_2d()->set_y(v);
+    }
+  }
+}
+
+void TensorsToObjectsCalculator::AssignObjectIdAndTimestamp(
+    int64 timestamp_us, FrameAnnotation* annotation) {
+  for (auto& ann : *annotation->mutable_annotations()) {
+    ann.set_object_id(GetNextObjectId());
+  }
+  annotation->set_timestamp(timestamp_us);
+}
+
+}  // namespace mediapipe

mediapipe/modules/objectron/calculators/tensors_to_objects_calculator.proto

@@ -0,0 +1,39 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// The option proto for the TensorsToObjectsCalculatorOptions.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+import "mediapipe/modules/objectron/calculators/belief_decoder_config.proto";
+
+message TensorsToObjectsCalculatorOptions {
+  extend CalculatorOptions {
+    optional TensorsToObjectsCalculatorOptions ext = 334361940;
+  }
+
+  // The number of output classes predicted by the detection model.
+  optional int32 num_classes = 1;
+
+  // The number of predicted keypoints.
+  optional int32 num_keypoints = 2;
+  // The dimension of each keypoint, e.g. number of values predicted for each
+  // keypoint.
+  optional int32 num_values_per_keypoint = 3 [default = 2];
+
+  optional BeliefDecoderConfig decoder_config = 4;
+}

mediapipe/modules/objectron/calculators/tflite_tensors_to_objects_calculator.cc

@@ -0,0 +1,216 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <memory>
+#include <unordered_map>
+#include <vector>
+
+#include "Eigen/Dense"
+#include "absl/memory/memory.h"
+#include "absl/strings/str_format.h"
+#include "absl/types/span.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/deps/file_path.h"
+#include "mediapipe/framework/port/opencv_core_inc.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/modules/objectron/calculators/annotation_data.pb.h"
+#include "mediapipe/modules/objectron/calculators/belief_decoder_config.pb.h"
+#include "mediapipe/modules/objectron/calculators/decoder.h"
+#include "mediapipe/modules/objectron/calculators/tensor_util.h"
+#include "mediapipe/modules/objectron/calculators/tflite_tensors_to_objects_calculator.pb.h"
+#include "tensorflow/lite/interpreter.h"
+
+namespace {
+constexpr char kInputStreamTag[] = "TENSORS";
+constexpr char kOutputStreamTag[] = "ANNOTATIONS";
+
+// Each detection object will be assigned an unique id that starts from 1.
+static int object_id = 0;
+
+inline int GetNextObjectId() { return ++object_id; }
+}  // namespace
+
+namespace mediapipe {
+
+// Convert result TFLite tensors from deep pursuit 3d model into
+// FrameAnnotation.
+//
+// Input:
+//  TENSORS - Vector of TfLiteTensor of type kTfLiteFloat32.
+// Output:
+//  ANNOTATIONS - Result FrameAnnotation.
+//
+// Usage example:
+// node {
+//   calculator: "TfLiteTensorsToObjectsCalculator"
+//   input_stream: "TENSORS:tensors"
+//   output_stream: "ANNOTATIONS:annotations"
+// }
+class TfLiteTensorsToObjectsCalculator : public CalculatorBase {
+ public:
+  static mediapipe::Status GetContract(CalculatorContract* cc);
+
+  mediapipe::Status Open(CalculatorContext* cc) override;
+  mediapipe::Status Process(CalculatorContext* cc) override;
+  mediapipe::Status Close(CalculatorContext* cc) override;
+
+ private:
+  mediapipe::Status ProcessCPU(CalculatorContext* cc,
+                               FrameAnnotation* output_objects);
+  mediapipe::Status LoadOptions(CalculatorContext* cc);
+  // Takes point_3d in FrameAnnotation, projects to 2D, and overwrite the
+  // point_2d field with the projection.
+  void Project3DTo2D(bool portrait, FrameAnnotation* annotation) const;
+  // Increment and assign object ID for each detected object.
+  // In a single MediaPipe session, the IDs are unique.
+  // Also assign timestamp for the FrameAnnotation to be the input packet
+  // timestamp.
+  void AssignObjectIdAndTimestamp(int64 timestamp_us,
+                                  FrameAnnotation* annotation);
+
+  int num_classes_ = 0;
+  int num_keypoints_ = 0;
+
+  ::mediapipe::TfLiteTensorsToObjectsCalculatorOptions options_;
+  std::unique_ptr<Decoder> decoder_;
+  Eigen::Matrix<float, 4, 4, Eigen::RowMajor> projection_matrix_;
+};
+REGISTER_CALCULATOR(TfLiteTensorsToObjectsCalculator);
+
+mediapipe::Status TfLiteTensorsToObjectsCalculator::GetContract(
+    CalculatorContract* cc) {
+  RET_CHECK(!cc->Inputs().GetTags().empty());
+  RET_CHECK(!cc->Outputs().GetTags().empty());
+
+  if (cc->Inputs().HasTag(kInputStreamTag)) {
+    cc->Inputs().Tag(kInputStreamTag).Set<std::vector<TfLiteTensor>>();
+  }
+
+  if (cc->Outputs().HasTag(kOutputStreamTag)) {
+    cc->Outputs().Tag(kOutputStreamTag).Set<FrameAnnotation>();
+  }
+  return mediapipe::OkStatus();
+}
+
+mediapipe::Status TfLiteTensorsToObjectsCalculator::Open(
+    CalculatorContext* cc) {
+  MP_RETURN_IF_ERROR(LoadOptions(cc));
+  // clang-format off
+  projection_matrix_ <<
+      1.5731,   0,       0, 0,
+      0,   2.0975,       0, 0,
+      0,        0, -1.0002, -0.2,
+      0,        0,      -1, 0;
+  // clang-format on
+  decoder_ = absl::make_unique<Decoder>(
+      BeliefDecoderConfig(options_.decoder_config()));
+
+  return mediapipe::OkStatus();
+}
+
+mediapipe::Status TfLiteTensorsToObjectsCalculator::Process(
+    CalculatorContext* cc) {
+  if (cc->Inputs().Tag(kInputStreamTag).IsEmpty()) {
+    return mediapipe::OkStatus();
+  }
+
+  auto output_objects = absl::make_unique<FrameAnnotation>();
+
+  MP_RETURN_IF_ERROR(ProcessCPU(cc, output_objects.get()));
+
+  // Output
+  if (cc->Outputs().HasTag(kOutputStreamTag)) {
+    cc->Outputs()
+        .Tag(kOutputStreamTag)
+        .Add(output_objects.release(), cc->InputTimestamp());
+  }
+
+  return mediapipe::OkStatus();
+}
+
+mediapipe::Status TfLiteTensorsToObjectsCalculator::ProcessCPU(
+    CalculatorContext* cc, FrameAnnotation* output_objects) {
+  const auto& input_tensors =
+      cc->Inputs().Tag(kInputStreamTag).Get<std::vector<TfLiteTensor>>();
+
+  cv::Mat prediction_heatmap = ConvertTfliteTensorToCvMat(input_tensors[0]);
+  cv::Mat offsetmap = ConvertTfliteTensorToCvMat(input_tensors[1]);
+
+  *output_objects =
+      decoder_->DecodeBoundingBoxKeypoints(prediction_heatmap, offsetmap);
+  auto status = decoder_->Lift2DTo3D(projection_matrix_, /*portrait*/ true,
+                                     output_objects);
+  if (!status.ok()) {
+    LOG(ERROR) << status;
+    return status;
+  }
+  Project3DTo2D(/*portrait*/ true, output_objects);
+  AssignObjectIdAndTimestamp(cc->InputTimestamp().Microseconds(),
+                             output_objects);
+
+  return mediapipe::OkStatus();
+}
+
+mediapipe::Status TfLiteTensorsToObjectsCalculator::Close(
+    CalculatorContext* cc) {
+  return mediapipe::OkStatus();
+}
+
+mediapipe::Status TfLiteTensorsToObjectsCalculator::LoadOptions(
+    CalculatorContext* cc) {
+  // Get calculator options specified in the graph.
+  options_ =
+      cc->Options<::mediapipe::TfLiteTensorsToObjectsCalculatorOptions>();
+
+  num_classes_ = options_.num_classes();
+  num_keypoints_ = options_.num_keypoints();
+
+  // Currently only support 2D when num_values_per_keypoint equals to 2.
+  CHECK_EQ(options_.num_values_per_keypoint(), 2);
+
+  return mediapipe::OkStatus();
+}
+
+void TfLiteTensorsToObjectsCalculator::Project3DTo2D(
+    bool portrait, FrameAnnotation* annotation) const {
+  for (auto& ann : *annotation->mutable_annotations()) {
+    for (auto& key_point : *ann.mutable_keypoints()) {
+      Eigen::Vector4f point3d;
+      point3d << key_point.point_3d().x(), key_point.point_3d().y(),
+          key_point.point_3d().z(), 1.0f;
+      Eigen::Vector4f point3d_projection = projection_matrix_ * point3d;
+      float u, v;
+      const float inv_w = 1.0f / point3d_projection(3);
+      if (portrait) {
+        u = (point3d_projection(1) * inv_w + 1.0f) * 0.5f;
+        v = (point3d_projection(0) * inv_w + 1.0f) * 0.5f;
+      } else {
+        u = (point3d_projection(0) * inv_w + 1.0f) * 0.5f;
+        v = (1.0f - point3d_projection(1) * inv_w) * 0.5f;
+      }
+      key_point.mutable_point_2d()->set_x(u);
+      key_point.mutable_point_2d()->set_y(v);
+    }
+  }
+}
+
+void TfLiteTensorsToObjectsCalculator::AssignObjectIdAndTimestamp(
+    int64 timestamp_us, FrameAnnotation* annotation) {
+  for (auto& ann : *annotation->mutable_annotations()) {
+    ann.set_object_id(GetNextObjectId());
+  }
+  annotation->set_timestamp(timestamp_us);
+}
+
+}  // namespace mediapipe

mediapipe/modules/objectron/calculators/tflite_tensors_to_objects_calculator.proto

@@ -0,0 +1,39 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// The option proto for the TfLiteTensorsToObjectsCalculatorOptions.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+import "mediapipe/modules/objectron/calculators/belief_decoder_config.proto";
+
+message TfLiteTensorsToObjectsCalculatorOptions {
+  extend CalculatorOptions {
+    optional TfLiteTensorsToObjectsCalculatorOptions ext = 263667646;
+  }
+
+  // The number of output classes predicted by the detection model.
+  optional int32 num_classes = 1;
+
+  // The number of predicted keypoints.
+  optional int32 num_keypoints = 2;
+  // The dimension of each keypoint, e.g. number of values predicted for each
+  // keypoint.
+  optional int32 num_values_per_keypoint = 3 [default = 2];
+
+  optional BeliefDecoderConfig decoder_config = 4;
+}

mediapipe/modules/objectron/calculators/types.h

@@ -0,0 +1,56 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_MODULES_OBJECTRON_CALCULATORS_TYPES_H_
+#define MEDIAPIPE_MODULES_OBJECTRON_CALCULATORS_TYPES_H_
+
+#include <array>
+
+#include "Eigen/Geometry"
+
+namespace mediapipe {
+
+using Eigen::Map;
+using Eigen::Vector2f;
+using Eigen::Vector3f;
+using Eigen::Vector4f;
+using Matrix4f_RM = Eigen::Matrix<float, 4, 4, Eigen::RowMajor>;
+using Matrix3f_RM = Eigen::Matrix<float, 3, 3, Eigen::RowMajor>;
+
+using Face = std::array<int, 4>;
+
+struct SuperPoint {
+  enum PointSourceType { kPointCloud = 0, kBoundingBox = 1, kSkeleton = 2 };
+  // The id of the point in the point-cloud
+  int reference_point;
+  // The source of the
+  PointSourceType source;
+  // The id of the point in set of points in current frame
+  int id;
+  // If source is kBoundingBox or kSkeleton, object_id stores the id of which \
+  // object this point belongs to.
+  int object_id;
+  // projected u-v value
+  Vector2f uv;
+  Vector2f pixel;
+  // the 3D point
+  Vector3f point_3d;
+  // Color
+  Eigen::Matrix<unsigned char, 4, 1> color;
+  bool rendered;
+};
+
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_MODULES_OBJECTRON_CALCULATORS_TYPES_H_

mediapipe/modules/objectron/object_detection_oid_v4_cpu.pbtxt

@@ -0,0 +1,177 @@
+# MediaPipe Objectron object detection CPU subgraph.
+
+type: "ObjectDetectionOidV4Subgraph"
+
+input_stream: "IMAGE:input_video"
+input_side_packet: "LABELS_CSV:allowed_labels"
+output_stream: "NORM_RECT:box_rect_from_object_detections"
+
+# Transforms the input image on GPU to a 300x300 image. To scale the image, by
+# default it uses the STRETCH scale mode that maps the entire input image to the
+# entire transformed image. As a result, image aspect ratio may be changed and
+# objects in the image may be deformed (stretched or squeezed), but the object
+# detection model used in this graph is agnostic to that deformation.
+node: {
+  calculator: "ImageTransformationCalculator"
+  input_stream: "IMAGE:input_video"
+  output_stream: "IMAGE:transformed_input_video"
+  options: {
+    [mediapipe.ImageTransformationCalculatorOptions.ext] {
+      output_width: 300
+      output_height: 300
+    }
+  }
+}
+
+# Converts the transformed input image on GPU into an image tensor stored as a
+# TfLiteTensor.
+node {
+  calculator: "TfLiteConverterCalculator"
+  input_stream: "IMAGE:transformed_input_video"
+  output_stream: "TENSORS:image_tensor"
+}
+
+# Runs a TensorFlow Lite model on GPU that takes an image tensor and outputs a
+# vector of tensors representing, for instance, detection boxes/keypoints and
+# scores.
+node {
+  calculator: "TfLiteInferenceCalculator"
+  input_stream: "TENSORS:image_tensor"
+  output_stream: "TENSORS:detection_tensors"
+  options: {
+    [mediapipe.TfLiteInferenceCalculatorOptions.ext] {
+      model_path: "mediapipe/models/object_detection_ssd_mobilenetv2_oidv4_fp16.tflite"
+    }
+  }
+}
+
+# Generates a single side packet containing a vector of SSD anchors based on
+# the specification in the options.
+node {
+  calculator: "SsdAnchorsCalculator"
+  output_side_packet: "anchors"
+  options: {
+    [mediapipe.SsdAnchorsCalculatorOptions.ext] {
+      num_layers: 6
+      min_scale: 0.2
+      max_scale: 0.95
+      input_size_height: 300
+      input_size_width: 300
+      anchor_offset_x: 0.5
+      anchor_offset_y: 0.5
+      strides: 16
+      strides: 32
+      strides: 64
+      strides: 128
+      strides: 256
+      strides: 512
+      aspect_ratios: 1.0
+      aspect_ratios: 2.0
+      aspect_ratios: 0.5
+      aspect_ratios: 3.0
+      aspect_ratios: 0.3333
+      reduce_boxes_in_lowest_layer: true
+    }
+  }
+}
+
+# Decodes the detection tensors generated by the TensorFlow Lite model, based on
+# the SSD anchors and the specification in the options, into a vector of
+# detections. Each detection describes a detected object.
+node {
+  calculator: "TfLiteTensorsToDetectionsCalculator"
+  input_stream: "TENSORS:detection_tensors"
+  input_side_packet: "ANCHORS:anchors"
+  output_stream: "DETECTIONS:detections"
+  options: {
+    [mediapipe.TfLiteTensorsToDetectionsCalculatorOptions.ext] {
+      num_classes: 195
+      num_boxes: 1917
+      num_coords: 4
+      ignore_classes: 0
+      sigmoid_score: true
+      apply_exponential_on_box_size: true
+      x_scale: 10.0
+      y_scale: 10.0
+      h_scale: 5.0
+      w_scale: 5.0
+      min_score_thresh: 0.6
+    }
+  }
+}
+
+# Performs non-max suppression to remove excessive detections.
+node {
+  calculator: "NonMaxSuppressionCalculator"
+  input_stream: "detections"
+  output_stream: "suppressed_detections"
+  options: {
+    [mediapipe.NonMaxSuppressionCalculatorOptions.ext] {
+      min_suppression_threshold: 0.4
+      max_num_detections: 1
+      overlap_type: INTERSECTION_OVER_UNION
+      return_empty_detections: true
+    }
+  }
+}
+
+# Maps detection label IDs to the corresponding label text. The label map is
+# provided in the label_map_path option.
+node {
+  calculator: "DetectionLabelIdToTextCalculator"
+  input_stream: "suppressed_detections"
+  output_stream: "labeled_detections"
+  options: {
+    [mediapipe.DetectionLabelIdToTextCalculatorOptions.ext] {
+      label_map_path: "mediapipe/models/object_detection_oidv4_labelmap.pbtxt"
+    }
+  }
+}
+
+node {
+  calculator: "FilterDetectionCalculator"
+  input_stream: "DETECTIONS:labeled_detections"
+  output_stream: "DETECTIONS:filtered_detections"
+  input_side_packet: "LABELS_CSV:allowed_labels"
+  options: {
+    [mediapipe.FilterDetectionCalculatorOptions.ext]: {
+      min_score: 0.4
+    }
+  }
+}
+
+# Extracts image size from the input images.
+node {
+  calculator: "ImagePropertiesCalculator"
+  input_stream: "IMAGE:input_video"
+  output_stream: "SIZE:image_size"
+}
+
+# Converts results of box detection into a rectangle (normalized by image size)
+# that encloses the box.
+node {
+  calculator: "DetectionsToRectsCalculator"
+  input_stream: "DETECTIONS:filtered_detections"
+  input_stream: "IMAGE_SIZE:image_size"
+  output_stream: "NORM_RECT:box_rect"
+  options: {
+    [mediapipe.DetectionsToRectsCalculatorOptions.ext] {
+      output_zero_rect_for_empty_detections: true
+    }
+  }
+}
+
+# Expands the rectangle that contains the box so that it's likely to cover the
+# entire box.
+node {
+  calculator: "RectTransformationCalculator"
+  input_stream: "NORM_RECT:box_rect"
+  input_stream: "IMAGE_SIZE:image_size"
+  output_stream: "box_rect_from_object_detections"
+  options: {
+    [mediapipe.RectTransformationCalculatorOptions.ext] {
+      scale_x: 1.5
+      scale_y: 1.5
+    }
+  }
+}

mediapipe/modules/objectron/object_detection_oid_v4_gpu.pbtxt

@@ -0,0 +1,177 @@
+# MediaPipe Objectron object detection GPU subgraph.
+
+type: "ObjectDetectionOidV4Subgraph"
+
+input_stream: "IMAGE_GPU:input_video"
+input_side_packet: "LABELS_CSV:allowed_labels"
+output_stream: "NORM_RECT:box_rect_from_object_detections"
+
+# Transforms the input image on GPU to a 300x300 image. To scale the image, by
+# default it uses the STRETCH scale mode that maps the entire input image to the
+# entire transformed image. As a result, image aspect ratio may be changed and
+# objects in the image may be deformed (stretched or squeezed), but the object
+# detection model used in this graph is agnostic to that deformation.
+node: {
+  calculator: "ImageTransformationCalculator"
+  input_stream: "IMAGE_GPU:input_video"
+  output_stream: "IMAGE_GPU:transformed_input_video"
+  options: {
+    [mediapipe.ImageTransformationCalculatorOptions.ext] {
+      output_width: 300
+      output_height: 300
+    }
+  }
+}
+
+# Converts the transformed input image on GPU into an image tensor stored as a
+# TfLiteTensor.
+node {
+  calculator: "TfLiteConverterCalculator"
+  input_stream: "IMAGE_GPU:transformed_input_video"
+  output_stream: "TENSORS_GPU:image_tensor"
+}
+
+# Runs a TensorFlow Lite model on GPU that takes an image tensor and outputs a
+# vector of tensors representing, for instance, detection boxes/keypoints and
+# scores.
+node {
+  calculator: "TfLiteInferenceCalculator"
+  input_stream: "TENSORS_GPU:image_tensor"
+  output_stream: "TENSORS_GPU:detection_tensors"
+  options: {
+    [mediapipe.TfLiteInferenceCalculatorOptions.ext] {
+      model_path: "mediapipe/models/object_detection_ssd_mobilenetv2_oidv4_fp16.tflite"
+    }
+  }
+}
+
+# Generates a single side packet containing a vector of SSD anchors based on
+# the specification in the options.
+node {
+  calculator: "SsdAnchorsCalculator"
+  output_side_packet: "anchors"
+  options: {
+    [mediapipe.SsdAnchorsCalculatorOptions.ext] {
+      num_layers: 6
+      min_scale: 0.2
+      max_scale: 0.95
+      input_size_height: 300
+      input_size_width: 300
+      anchor_offset_x: 0.5
+      anchor_offset_y: 0.5
+      strides: 16
+      strides: 32
+      strides: 64
+      strides: 128
+      strides: 256
+      strides: 512
+      aspect_ratios: 1.0
+      aspect_ratios: 2.0
+      aspect_ratios: 0.5
+      aspect_ratios: 3.0
+      aspect_ratios: 0.3333
+      reduce_boxes_in_lowest_layer: true
+    }
+  }
+}
+
+# Decodes the detection tensors generated by the TensorFlow Lite model, based on
+# the SSD anchors and the specification in the options, into a vector of
+# detections. Each detection describes a detected object.
+node {
+  calculator: "TfLiteTensorsToDetectionsCalculator"
+  input_stream: "TENSORS_GPU:detection_tensors"
+  input_side_packet: "ANCHORS:anchors"
+  output_stream: "DETECTIONS:detections"
+  options: {
+    [mediapipe.TfLiteTensorsToDetectionsCalculatorOptions.ext] {
+      num_classes: 195
+      num_boxes: 1917
+      num_coords: 4
+      ignore_classes: 0
+      sigmoid_score: true
+      apply_exponential_on_box_size: true
+      x_scale: 10.0
+      y_scale: 10.0
+      h_scale: 5.0
+      w_scale: 5.0
+      min_score_thresh: 0.6
+    }
+  }
+}
+
+# Performs non-max suppression to remove excessive detections.
+node {
+  calculator: "NonMaxSuppressionCalculator"
+  input_stream: "detections"
+  output_stream: "suppressed_detections"
+  options: {
+    [mediapipe.NonMaxSuppressionCalculatorOptions.ext] {
+      min_suppression_threshold: 0.4
+      max_num_detections: 1
+      overlap_type: INTERSECTION_OVER_UNION
+      return_empty_detections: true
+    }
+  }
+}
+
+# Maps detection label IDs to the corresponding label text. The label map is
+# provided in the label_map_path option.
+node {
+  calculator: "DetectionLabelIdToTextCalculator"
+  input_stream: "suppressed_detections"
+  output_stream: "labeled_detections"
+  options: {
+    [mediapipe.DetectionLabelIdToTextCalculatorOptions.ext] {
+      label_map_path: "mediapipe/models/object_detection_oidv4_labelmap.pbtxt"
+    }
+  }
+}
+
+node {
+  calculator: "FilterDetectionCalculator"
+  input_stream: "DETECTIONS:labeled_detections"
+  output_stream: "DETECTIONS:filtered_detections"
+  input_side_packet: "LABELS_CSV:allowed_labels"
+  options: {
+    [mediapipe.FilterDetectionCalculatorOptions.ext]: {
+      min_score: 0.4
+    }
+  }
+}
+
+# Extracts image size from the input images.
+node {
+  calculator: "ImagePropertiesCalculator"
+  input_stream: "IMAGE_GPU:input_video"
+  output_stream: "SIZE:image_size"
+}
+
+# Converts results of box detection into a rectangle (normalized by image size)
+# that encloses the box.
+node {
+  calculator: "DetectionsToRectsCalculator"
+  input_stream: "DETECTIONS:filtered_detections"
+  input_stream: "IMAGE_SIZE:image_size"
+  output_stream: "NORM_RECT:box_rect"
+  options: {
+    [mediapipe.DetectionsToRectsCalculatorOptions.ext] {
+      output_zero_rect_for_empty_detections: true
+    }
+  }
+}
+
+# Expands the rectangle that contains the box so that it's likely to cover the
+# entire box.
+node {
+  calculator: "RectTransformationCalculator"
+  input_stream: "NORM_RECT:box_rect"
+  input_stream: "IMAGE_SIZE:image_size"
+  output_stream: "box_rect_from_object_detections"
+  options: {
+    [mediapipe.RectTransformationCalculatorOptions.ext] {
+      scale_x: 1.5
+      scale_y: 1.5
+    }
+  }
+}

mediapipe/modules/objectron/objectron_cpu.pbtxt

@@ -0,0 +1,112 @@
+# MediaPipe Objectron on CPU that produces 3D bounding boxes for objects.
+input_stream: "IMAGE:input_video"
+# TfLite model for 3D bounding box landmark prediction
+input_side_packet: "MODEL:box_landmark_model"
+# Allowed category labels, e.g. Footwear, Coffee cup, Mug, Chair, Camera
+input_side_packet: "LABELS_CSV:allowed_labels"
+# Bounding box landmarks topology definition.
+# The numbers are indices in the box_landmarks list.
+#
+#       3 + + + + + + + + 7
+#       +\                +\          UP
+#       + \               + \
+#       +  \              +  \        |
+#       +   4 + + + + + + + + 8       | y
+#       +   +             +   +       |
+#       +   +             +   +       |
+#       +   +     (0)     +   +       .------- x
+#       +   +             +   +        \
+#       1 + + + + + + + + 5   +         \
+#        \  +              \  +          \ z
+#         \ +               \ +           \
+#          \+                \+
+#           2 + + + + + + + + 6
+#
+output_stream: "LANDMARKS:box_landmarks"
+# Crop rectangle derived from bounding box landmarks.
+output_stream: "NORM_RECT:box_rect"
+
+
+# Caches a box-presence decision fed back from boxLandmarkSubgraph, and upon
+# the arrival of the next input image sends out the cached decision with the
+# timestamp replaced by that of the input image, essentially generating a packet
+# that carries the previous box-presence decision. Note that upon the arrival
+# of the very first input image, an empty packet is sent out to jump start the
+# feedback loop.
+node {
+  calculator: "PreviousLoopbackCalculator"
+  input_stream: "MAIN:input_video"
+  input_stream: "LOOP:box_presence"
+  input_stream_info: {
+    tag_index: "LOOP"
+    back_edge: true
+  }
+  output_stream: "PREV_LOOP:prev_box_presence"
+}
+
+# Drops the incoming image if boxLandmarkSubgraph was able to identify box
+# presence in the previous image. Otherwise, passes the incoming image through
+# to trigger a new round of box detection in boxDetectionSubgraph.
+node {
+  calculator: "GateCalculator"
+  input_stream: "input_video"
+  input_stream: "DISALLOW:prev_box_presence"
+  output_stream: "box_detection_input_video"
+
+  options: {
+    [mediapipe.GateCalculatorOptions.ext] {
+      empty_packets_as_allow: true
+    }
+  }
+}
+
+# Subgraph that detections boxs (see object_detection_oid_v4_cpu.pbtxt).
+node {
+  calculator: "ObjectDetectionOidV4Subgraph"
+  input_stream: "IMAGE:box_detection_input_video"
+  input_side_packet: "LABELS_CSV:allowed_labels"
+  output_stream: "NORM_RECT:box_rect_from_object_detections"
+}
+
+# Subgraph that localizes box landmarks (see box_landmark_gpu.pbtxt).
+node {
+  calculator: "BoxLandmarkSubgraph"
+  input_stream: "IMAGE:input_video"
+  input_stream: "NORM_RECT:box_rect"
+  input_side_packet: "MODEL:box_landmark_model"
+  output_stream: "LANDMARKS:box_landmarks"
+  output_stream: "NORM_RECT:box_rect_from_landmarks"
+  output_stream: "PRESENCE:box_presence"
+}
+
+# Caches a box rectangle fed back from boxLandmarkSubgraph, and upon the
+# arrival of the next input image sends out the cached rectangle with the
+# timestamp replaced by that of the input image, essentially generating a packet
+# that carries the previous box rectangle. Note that upon the arrival of the
+# very first input image, an empty packet is sent out to jump start the
+# feedback loop.
+node {
+  calculator: "PreviousLoopbackCalculator"
+  input_stream: "MAIN:input_video"
+  input_stream: "LOOP:box_rect_from_landmarks"
+  input_stream_info: {
+    tag_index: "LOOP"
+    back_edge: true
+  }
+  output_stream: "PREV_LOOP:prev_box_rect_from_landmarks"
+}
+
+# Merges a stream of box rectangles generated by ObjectDetectionSubgraph and that
+# generated by BoxLandmarkSubgraph into a single output stream by selecting
+# between one of the two streams. The former is selected if the incoming packet
+# is not empty, i.e., box detection is performed on the current image by
+# BoxDetectionSubgraph (because BoxLandmarkSubgraph could not identify box
+# presence in the previous image). Otherwise, the latter is selected, which is
+# never empty because BoxLandmarkSubgraphs processes all images (that went
+# through FlowLimiterCaculator).
+node {
+  calculator: "MergeCalculator"
+  input_stream: "box_rect_from_object_detections"
+  input_stream: "prev_box_rect_from_landmarks"
+  output_stream: "box_rect"
+}

mediapipe/modules/objectron/objectron_detection_1stage_gpu.pbtxt

@@ -0,0 +1,81 @@
+# MediaPipe Objectron detection gpu subgraph
+
+type: "ObjectronDetectionSubgraphGpu"
+
+input_stream: "IMAGE_GPU:input_video"
+output_stream: "ANNOTATIONS:objects"
+
+# Transforms the input image on GPU to a 480x640 image. To scale the input
+# image, the scale_mode option is set to FIT to preserve the aspect ratio,
+# resulting in potential letterboxing in the transformed image.
+node: {
+  calculator: "ImageTransformationCalculator"
+  input_stream: "IMAGE_GPU:input_video"
+  output_stream: "IMAGE_GPU:transformed_input_video"
+  options: {
+    [mediapipe.ImageTransformationCalculatorOptions.ext] {
+      output_width: 480
+      output_height: 640
+      scale_mode: FIT
+    }
+  }
+}
+
+# Converts the transformed input image on GPU into an image tensor stored as a
+# TfLiteTensor.
+node {
+  calculator: "TfLiteConverterCalculator"
+  input_stream: "IMAGE_GPU:transformed_input_video"
+  output_stream: "TENSORS_GPU:image_tensor"
+}
+
+# Generates a single side packet containing a TensorFlow Lite op resolver that
+# supports custom ops needed by the model used in this graph.
+node {
+  calculator: "TfLiteCustomOpResolverCalculator"
+  output_side_packet: "opresolver"
+  options: {
+    [mediapipe.TfLiteCustomOpResolverCalculatorOptions.ext] {
+      use_gpu: true
+    }
+  }
+}
+
+# Runs a TensorFlow Lite model on GPU that takes an image tensor and outputs a
+# vector of tensors representing, for instance, detection boxes/keypoints and
+# scores.
+node {
+  calculator: "TfLiteInferenceCalculator"
+  input_stream: "TENSORS_GPU:image_tensor"
+  output_stream: "TENSORS:detection_tensors"
+  input_side_packet: "CUSTOM_OP_RESOLVER:opresolver"
+  options: {
+    [mediapipe.TfLiteInferenceCalculatorOptions.ext] {
+      model_path: "object_detection_3d.tflite"
+    }
+  }
+}
+
+# Decodes the model's output tensor (the heatmap and the distance fields) to 2D
+# keypoints. There are nine 2D keypoints: one center keypoint and eight vertices
+# for the 3D bounding box. The calculator parameters determine's the decoder's
+# sensitivity.
+node {
+  calculator: "TfLiteTensorsToObjectsCalculator"
+  input_stream: "TENSORS:detection_tensors"
+  output_stream: "ANNOTATIONS:objects"
+  options: {
+    [mediapipe.TfLiteTensorsToObjectsCalculatorOptions.ext] {
+      num_classes: 1
+      num_keypoints: 9
+      decoder_config {
+        heatmap_threshold: 0.6
+        local_max_distance: 2
+        offset_scale_coef: 1.0
+        voting_radius: 2
+        voting_allowance: 1
+        voting_threshold: 0.2
+      }
+    }
+  }
+}

mediapipe/modules/objectron/objectron_gpu.pbtxt

@@ -0,0 +1,92 @@
+# MediaPipe Objectron on GPU that produces 3D bounding boxes for objects.
+
+# Input/Output streams and input side packets.
+# Note that the input video is assumed to have aspect ratio 3:4 (width:height).
+input_stream: "IMAGE_GPU:input_video"
+# Allowed category labels, e.g. Footwear, Coffee cup, Mug, Chair, Camera
+input_side_packet: "LABELS_CSV:allowed_labels"
+output_stream: "FRAME_ANNOTATION:lifted_objects"
+
+
+# Caches a box-presence decision fed back from boxLandmarkSubgraph, and upon
+# the arrival of the next input image sends out the cached decision with the
+# timestamp replaced by that of the input image, essentially generating a packet
+# that carries the previous box-presence decision. Note that upon the arrival
+# of the very first input image, an empty packet is sent out to jump start the
+# feedback loop.
+node {
+  calculator: "PreviousLoopbackCalculator"
+  input_stream: "MAIN:input_video"
+  input_stream: "LOOP:box_presence"
+  input_stream_info: {
+    tag_index: "LOOP"
+    back_edge: true
+  }
+  output_stream: "PREV_LOOP:prev_box_presence"
+}
+
+# Drops the incoming image if boxLandmarkSubgraph was able to identify box
+# presence in the previous image. Otherwise, passes the incoming image through
+# to trigger a new round of box detection in boxDetectionSubgraph.
+node {
+  calculator: "GateCalculator"
+  input_stream: "input_video"
+  input_stream: "DISALLOW:prev_box_presence"
+  output_stream: "detection_input_video"
+
+  options: {
+    [mediapipe.GateCalculatorOptions.ext] {
+      empty_packets_as_allow: true
+    }
+  }
+}
+
+# Subgraph that performs 2D object detection.
+node {
+  calculator: "ObjectDetectionOidV4Subgraph"
+  input_stream: "IMAGE_GPU:detection_input_video"
+  input_side_packet: "LABELS_CSV:allowed_labels"
+  output_stream: "NORM_RECT:box_rect_from_object_detections"
+}
+
+# Subgraph that localizes box landmarks.
+node {
+  calculator: "BoxLandmarkSubgraph"
+  input_stream: "IMAGE:input_video"
+  input_stream: "NORM_RECT:box_rect"
+  output_stream: "FRAME_ANNOTATION:lifted_objects"
+  output_stream: "NORM_RECT:box_rect_from_landmarks"
+  output_stream: "PRESENCE:box_presence"
+}
+
+# Caches a box rectangle fed back from boxLandmarkSubgraph, and upon the
+# arrival of the next input image sends out the cached rectangle with the
+# timestamp replaced by that of the input image, essentially generating a packet
+# that carries the previous box rectangle. Note that upon the arrival of the
+# very first input image, an empty packet is sent out to jump start the
+# feedback loop.
+node {
+  calculator: "PreviousLoopbackCalculator"
+  input_stream: "MAIN:input_video"
+  input_stream: "LOOP:box_rect_from_landmarks"
+  input_stream_info: {
+    tag_index: "LOOP"
+    back_edge: true
+  }
+  output_stream: "PREV_LOOP:prev_box_rect_from_landmarks"
+}
+
+# Merges a stream of box rectangles generated by boxDetectionSubgraph and that
+# generated by boxLandmarkSubgraph into a single output stream by selecting
+# between one of the two streams. The former is selected if the incoming packet
+# is not empty, i.e., box detection is performed on the current image by
+# boxDetectionSubgraph (because boxLandmarkSubgraph could not identify box
+# presence in the previous image). Otherwise, the latter is selected, which is
+# never empty because boxLandmarkSubgraphs processes all images (that went
+# through FlowLimiterCaculator).
+node {
+  calculator: "MergeCalculator"
+  input_stream: "box_rect_from_object_detections"
+  input_stream: "prev_box_rect_from_landmarks"
+  output_stream: "box_rect"
+}

mediapipe/modules/objectron/objectron_tracking_1stage_gpu.pbtxt

@@ -0,0 +1,170 @@
+# MediaPipe Objectron tracking gpu subgraph
+
+type: "ObjectronTrackingSubgraphGpu"
+
+input_stream: "FRAME_ANNOTATION:objects"
+input_stream: "IMAGE_GPU:input_video"
+output_stream: "LIFTED_FRAME_ANNOTATION:lifted_tracked_objects"
+
+
+# Converts the detected keypoints to Boxes, used by the tracking subgraph.
+node {
+  calculator: "FrameAnnotationToTimedBoxListCalculator"
+  input_stream: "FRAME_ANNOTATION:objects"
+  output_stream: "BOXES:start_pos"
+}
+
+node: {
+  calculator: "ImageTransformationCalculator"
+  input_stream: "IMAGE_GPU:input_video"
+  output_stream: "IMAGE_GPU:downscaled_input_video"
+  options: {
+    [mediapipe.ImageTransformationCalculatorOptions.ext] {
+      output_width: 240
+      output_height: 320
+    }
+  }
+}
+
+# Converts GPU buffer to ImageFrame for processing tracking.
+node: {
+  calculator: "GpuBufferToImageFrameCalculator"
+  input_stream: "downscaled_input_video"
+  output_stream: "downscaled_input_video_cpu"
+}
+
+# Performs motion analysis on an incoming video stream.
+node: {
+  calculator: "MotionAnalysisCalculator"
+  input_stream: "VIDEO:downscaled_input_video_cpu"
+  output_stream: "CAMERA:camera_motion"
+  output_stream: "FLOW:region_flow"
+
+  options: {
+    [mediapipe.MotionAnalysisCalculatorOptions.ext]: {
+      analysis_options {
+        analysis_policy: ANALYSIS_POLICY_CAMERA_MOBILE
+        flow_options {
+          fast_estimation_min_block_size: 100
+          top_inlier_sets: 1
+          frac_inlier_error_threshold: 3e-3
+          downsample_mode: DOWNSAMPLE_TO_INPUT_SIZE
+          verification_distance: 5.0
+          verify_long_feature_acceleration: true
+          verify_long_feature_trigger_ratio: 0.1
+          tracking_options {
+            max_features: 500
+            adaptive_extraction_levels: 2
+            min_eig_val_settings {
+              adaptive_lowest_quality_level: 2e-4
+            }
+            klt_tracker_implementation: KLT_OPENCV
+          }
+        }
+      }
+    }
+  }
+}
+
+# Reads optical flow fields defined in
+# mediapipe/framework/formats/motion/optical_flow_field.h,
+# returns a VideoFrame with 2 channels (v_x and v_y), each channel is quantized
+# to 0-255.
+node: {
+  calculator: "FlowPackagerCalculator"
+  input_stream: "FLOW:region_flow"
+  input_stream: "CAMERA:camera_motion"
+  output_stream: "TRACKING:tracking_data"
+
+  options: {
+    [mediapipe.FlowPackagerCalculatorOptions.ext]: {
+      flow_packager_options: {
+        binary_tracking_data_support: false
+      }
+    }
+  }
+}
+
+# Tracks box positions over time.
+node: {
+  calculator: "BoxTrackerCalculator"
+  input_stream: "TRACKING:tracking_data"
+  input_stream: "TRACK_TIME:input_video"
+  input_stream: "START_POS:start_pos"
+  input_stream: "CANCEL_OBJECT_ID:cancel_object_id"
+  input_stream_info: {
+    tag_index: "CANCEL_OBJECT_ID"
+    back_edge: true
+  }
+  output_stream: "BOXES:boxes"
+
+  input_stream_handler {
+    input_stream_handler: "SyncSetInputStreamHandler"
+    options {
+      [mediapipe.SyncSetInputStreamHandlerOptions.ext] {
+        sync_set {
+          tag_index: "TRACKING"
+          tag_index: "TRACK_TIME"
+        }
+        sync_set {
+          tag_index: "START_POS"
+        }
+        sync_set {
+          tag_index: "CANCEL_OBJECT_ID"
+        }
+      }
+    }
+  }
+
+  options: {
+    [mediapipe.BoxTrackerCalculatorOptions.ext]: {
+      tracker_options: {
+        track_step_options {
+          track_object_and_camera: true
+          tracking_degrees: TRACKING_DEGREE_OBJECT_ROTATION_SCALE
+          inlier_spring_force: 0.0
+          static_motion_temporal_ratio: 3e-2
+        }
+      }
+      visualize_tracking_data: false
+      streaming_track_data_cache_size: 100
+    }
+  }
+}
+
+# Consolidates tracking and detection results.
+node {
+  calculator: "FrameAnnotationTrackerCalculator"
+  input_stream: "FRAME_ANNOTATION:objects"
+  input_stream: "TRACKED_BOXES:boxes"
+  output_stream: "TRACKED_FRAME_ANNOTATION:tracked_objects"
+  output_stream: "CANCEL_OBJECT_ID:cancel_object_id"
+  options: {
+    [mediapipe.FrameAnnotationTrackerCalculatorOptions.ext] {
+      img_width: 240
+      img_height: 320
+      iou_threshold: 0.1
+    }
+  }
+
+  input_stream_handler {
+    input_stream_handler: "SyncSetInputStreamHandler"
+    options {
+      [mediapipe.SyncSetInputStreamHandlerOptions.ext] {
+        sync_set {
+          tag_index: "FRAME_ANNOTATION"
+        }
+        sync_set {
+          tag_index: "TRACKED_BOXES"
+        }
+      }
+    }
+  }
+}
+
+# Lift the tracked 2D keypoints to 3D using EPnP algorithm.
+node {
+  calculator: "Lift2DFrameAnnotationTo3DCalculator"
+  input_stream: "FRAME_ANNOTATION:tracked_objects"
+  output_stream: "LIFTED_FRAME_ANNOTATION:lifted_tracked_objects"
+}

mediapipe/modules/palm_detection/palm_detection_cpu.pbtxt

@@ -28,6 +28,7 @@ node {
         min: -1.0
         max: 1.0
       }
+      border_mode: BORDER_ZERO
     }
   }
 }

mediapipe/modules/palm_detection/palm_detection_gpu.pbtxt

@@ -28,6 +28,7 @@ node {
         min: -1.0
         max: 1.0
       }
+      border_mode: BORDER_ZERO
       gpu_origin: TOP_LEFT
     }
   }

mediapipe/modules/pose_detection/pose_detection_cpu.pbtxt

@@ -53,6 +53,7 @@ node: {
         min: -1.0
         max: 1.0
       }
+      border_mode: BORDER_ZERO
     }
   }
 }

mediapipe/modules/pose_detection/pose_detection_gpu.pbtxt

@@ -53,6 +53,7 @@ node: {
         min: -1.0
         max: 1.0
       }
+      border_mode: BORDER_ZERO
       gpu_origin: TOP_LEFT
     }
   }

mediapipe/modules/pose_landmark/BUILD

@@ -22,28 +22,23 @@ licenses(["notice"])
 package(default_visibility = ["//visibility:public"])
 
 mediapipe_simple_subgraph(
-    name = "pose_landmark_upper_body_by_roi_gpu",
-    graph = "pose_landmark_upper_body_by_roi_gpu.pbtxt",
-    register_as = "PoseLandmarkUpperBodyByRoiGpu",
+    name = "pose_landmark_model_loader",
+    graph = "pose_landmark_model_loader.pbtxt",
+    register_as = "PoseLandmarkModelLoader",
     deps = [
-        "//mediapipe/calculators/core:gate_calculator",
-        "//mediapipe/calculators/core:split_normalized_landmark_list_calculator",
-        "//mediapipe/calculators/core:split_vector_calculator",
-        "//mediapipe/calculators/tensor:image_to_tensor_calculator",
-        "//mediapipe/calculators/tensor:inference_calculator",
-        "//mediapipe/calculators/tensor:tensors_to_floats_calculator",
-        "//mediapipe/calculators/tensor:tensors_to_landmarks_calculator",
-        "//mediapipe/calculators/util:landmark_letterbox_removal_calculator",
-        "//mediapipe/calculators/util:landmark_projection_calculator",
-        "//mediapipe/calculators/util:thresholding_calculator",
+        "//mediapipe/calculators/core:constant_side_packet_calculator",
+        "//mediapipe/calculators/tflite:tflite_model_calculator",
+        "//mediapipe/calculators/util:local_file_contents_calculator",
+        "//mediapipe/framework/tool:switch_container",
     ],
 )
 
 mediapipe_simple_subgraph(
-    name = "pose_landmark_upper_body_by_roi_cpu",
-    graph = "pose_landmark_upper_body_by_roi_cpu.pbtxt",
-    register_as = "PoseLandmarkUpperBodyByRoiCpu",
+    name = "pose_landmark_by_roi_gpu",
+    graph = "pose_landmark_by_roi_gpu.pbtxt",
+    register_as = "PoseLandmarkByRoiGpu",
     deps = [
+        ":pose_landmark_model_loader",
         "//mediapipe/calculators/core:gate_calculator",
         "//mediapipe/calculators/core:split_normalized_landmark_list_calculator",
         "//mediapipe/calculators/core:split_vector_calculator",
@@ -54,6 +49,77 @@ mediapipe_simple_subgraph(
         "//mediapipe/calculators/util:landmark_letterbox_removal_calculator",
         "//mediapipe/calculators/util:landmark_projection_calculator",
         "//mediapipe/calculators/util:thresholding_calculator",
+        "//mediapipe/framework/tool:switch_container",
+    ],
+)
+
+mediapipe_simple_subgraph(
+    name = "pose_landmark_by_roi_cpu",
+    graph = "pose_landmark_by_roi_cpu.pbtxt",
+    register_as = "PoseLandmarkByRoiCpu",
+    deps = [
+        ":pose_landmark_model_loader",
+        "//mediapipe/calculators/core:gate_calculator",
+        "//mediapipe/calculators/core:split_normalized_landmark_list_calculator",
+        "//mediapipe/calculators/core:split_vector_calculator",
+        "//mediapipe/calculators/tensor:image_to_tensor_calculator",
+        "//mediapipe/calculators/tensor:inference_calculator",
+        "//mediapipe/calculators/tensor:tensors_to_floats_calculator",
+        "//mediapipe/calculators/tensor:tensors_to_landmarks_calculator",
+        "//mediapipe/calculators/util:landmark_letterbox_removal_calculator",
+        "//mediapipe/calculators/util:landmark_projection_calculator",
+        "//mediapipe/calculators/util:thresholding_calculator",
+        "//mediapipe/framework/tool:switch_container",
+    ],
+)
+
+mediapipe_simple_subgraph(
+    name = "pose_landmark_filtering",
+    graph = "pose_landmark_filtering.pbtxt",
+    register_as = "PoseLandmarkFiltering",
+    deps = [
+        "//mediapipe/calculators/util:landmarks_smoothing_calculator",
+        "//mediapipe/framework/tool:switch_container",
+    ],
+)
+
+mediapipe_simple_subgraph(
+    name = "pose_landmark_gpu",
+    graph = "pose_landmark_gpu.pbtxt",
+    register_as = "PoseLandmarkGpu",
+    deps = [
+        ":pose_detection_to_roi",
+        ":pose_landmark_by_roi_gpu",
+        ":pose_landmark_filtering",
+        ":pose_landmarks_to_roi",
+        "//mediapipe/calculators/core:constant_side_packet_calculator",
+        "//mediapipe/calculators/core:gate_calculator",
+        "//mediapipe/calculators/core:merge_calculator",
+        "//mediapipe/calculators/core:packet_presence_calculator",
+        "//mediapipe/calculators/core:previous_loopback_calculator",
+        "//mediapipe/calculators/core:split_vector_calculator",
+        "//mediapipe/calculators/image:image_properties_calculator",
+        "//mediapipe/modules/pose_detection:pose_detection_gpu",
+    ],
+)
+
+mediapipe_simple_subgraph(
+    name = "pose_landmark_cpu",
+    graph = "pose_landmark_cpu.pbtxt",
+    register_as = "PoseLandmarkCpu",
+    deps = [
+        ":pose_detection_to_roi",
+        ":pose_landmark_by_roi_cpu",
+        ":pose_landmark_filtering",
+        ":pose_landmarks_to_roi",
+        "//mediapipe/calculators/core:constant_side_packet_calculator",
+        "//mediapipe/calculators/core:gate_calculator",
+        "//mediapipe/calculators/core:merge_calculator",
+        "//mediapipe/calculators/core:packet_presence_calculator",
+        "//mediapipe/calculators/core:previous_loopback_calculator",
+        "//mediapipe/calculators/core:split_vector_calculator",
+        "//mediapipe/calculators/image:image_properties_calculator",
+        "//mediapipe/modules/pose_detection:pose_detection_cpu",
     ],
 )
 
@@ -62,17 +128,8 @@ mediapipe_simple_subgraph(
     graph = "pose_landmark_upper_body_gpu.pbtxt",
     register_as = "PoseLandmarkUpperBodyGpu",
     deps = [
-        ":pose_detection_to_roi",
-        ":pose_landmark_upper_body_by_roi_gpu",
-        ":pose_landmark_upper_body_landmarks_to_roi",
-        "//mediapipe/calculators/core:gate_calculator",
-        "//mediapipe/calculators/core:merge_calculator",
-        "//mediapipe/calculators/core:packet_presence_calculator",
-        "//mediapipe/calculators/core:previous_loopback_calculator",
-        "//mediapipe/calculators/core:split_vector_calculator",
-        "//mediapipe/calculators/image:image_properties_calculator",
-        "//mediapipe/calculators/util:logic_calculator",
-        "//mediapipe/modules/pose_detection:pose_detection_gpu",
+        ":pose_landmark_gpu",
+        "//mediapipe/calculators/core:constant_side_packet_calculator",
     ],
 )
 
@@ -81,44 +138,14 @@ mediapipe_simple_subgraph(
     graph = "pose_landmark_upper_body_cpu.pbtxt",
     register_as = "PoseLandmarkUpperBodyCpu",
     deps = [
-        ":pose_detection_to_roi",
-        ":pose_landmark_upper_body_by_roi_cpu",
-        ":pose_landmark_upper_body_landmarks_to_roi",
-        "//mediapipe/calculators/core:gate_calculator",
-        "//mediapipe/calculators/core:merge_calculator",
-        "//mediapipe/calculators/core:packet_presence_calculator",
-        "//mediapipe/calculators/core:previous_loopback_calculator",
-        "//mediapipe/calculators/core:split_vector_calculator",
-        "//mediapipe/calculators/image:image_properties_calculator",
-        "//mediapipe/calculators/util:logic_calculator",
-        "//mediapipe/modules/pose_detection:pose_detection_cpu",
-    ],
-)
-
-mediapipe_simple_subgraph(
-    name = "pose_landmark_upper_body_smoothed_cpu",
-    graph = "pose_landmark_upper_body_smoothed_cpu.pbtxt",
-    register_as = "PoseLandmarkUpperBodySmoothedCpu",
-    deps = [
-        ":pose_landmark_upper_body_cpu",
-        "//mediapipe/calculators/image:image_properties_calculator",
-        "//mediapipe/calculators/util:landmarks_smoothing_calculator",
-    ],
-)
-
-mediapipe_simple_subgraph(
-    name = "pose_landmark_upper_body_smoothed_gpu",
-    graph = "pose_landmark_upper_body_smoothed_gpu.pbtxt",
-    register_as = "PoseLandmarkUpperBodySmoothedGpu",
-    deps = [
-        ":pose_landmark_upper_body_gpu",
-        "//mediapipe/calculators/image:image_properties_calculator",
-        "//mediapipe/calculators/util:landmarks_smoothing_calculator",
+        ":pose_landmark_cpu",
+        "//mediapipe/calculators/core:constant_side_packet_calculator",
     ],
 )
 
 exports_files(
     srcs = [
+        "pose_landmark_full_body.tflite",
         "pose_landmark_upper_body.tflite",
     ],
 )
@@ -130,13 +157,14 @@ mediapipe_simple_subgraph(
     deps = [
         "//mediapipe/calculators/util:alignment_points_to_rects_calculator",
         "//mediapipe/calculators/util:rect_transformation_calculator",
+        "//mediapipe/framework/tool:switch_container",
     ],
 )
 
 mediapipe_simple_subgraph(
-    name = "pose_landmark_upper_body_landmarks_to_roi",
-    graph = "pose_landmark_upper_body_landmarks_to_roi.pbtxt",
-    register_as = "PoseLandmarkUpperBodyLandmarksToRoi",
+    name = "pose_landmarks_to_roi",
+    graph = "pose_landmarks_to_roi.pbtxt",
+    register_as = "PoseLandmarksToRoi",
     deps = [
         "//mediapipe/calculators/util:alignment_points_to_rects_calculator",
         "//mediapipe/calculators/util:landmarks_to_detection_calculator",

mediapipe/modules/pose_landmark/README.md

@@ -2,8 +2,9 @@
 
 Subgraphs|Details
 :--- | :---
-[`PoseLandmarkUpperBodyByRoiCpu`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/pose_landmark/pose_landmark_upper_body_by_roi_cpu.pbtxt)| Detects landmarks of a single pose. See landmarks (key points) [scheme](https://github.com/google/mediapipe/tree/master/mediapipe/modules/pose_landmark/pose_landmark_upper_body_topology.svg). (CPU input, and inference is executed on CPU.)
-[`PoseLandmarkUpperBodyByRoiGpu`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/pose_landmark/pose_landmark_upper_body_gpu.pbtxt)| Detects landmarks of a single pose. See landmarks (key points) [scheme](https://github.com/google/mediapipe/tree/master/mediapipe/modules/pose_landmark/pose_landmark_upper_body_topology.svg). (GPU input, and inference is executed on GPU)
-[`PoseLandmarkUpperBodyCpu`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/pose_landmark/pose_landmark_upper_body_cpu.pbtxt)| Detects and tracks landmarks of a single pose. See landmarks (key points) [scheme](https://github.com/google/mediapipe/tree/master/mediapipe/modules/pose_landmark/pose_landmark_upper_body_topology.svg). (CPU input, and inference is executed on CPU)
-[`PoseLandmarkUpperBodyGpu`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/pose_landmark/pose_landmark_upper_body_gpu.pbtxt)| Detects and tracks landmarks of a single pose. See landmarks (key points) [scheme](https://github.com/google/mediapipe/tree/master/mediapipe/modules/pose_landmark/pose_landmark_upper_body_topology.svg). (GPU input, and inference is executed on GPU.)
-
+[`PoseLandmarkByRoiCpu`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/pose_landmark/pose_landmark_by_roi_cpu.pbtxt)| Detects landmarks of a single body pose, full-body by default but can be configured (via an input side packet) to cover upper-body only. See landmarks (aka keypoints) [scheme](https://github.com/google/mediapipe/tree/master/mediapipe/modules/pose_landmark/pose_landmark_full_body_topology.svg). (CPU input, and inference is executed on CPU.)
+[`PoseLandmarkByRoiGpu`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/pose_landmark/pose_landmark_by_roi_gpu.pbtxt)| Detects landmarks of a single body pose, full-body by default but can be configured (via an input side packet) to cover upper-body only. See landmarks (aka keypoints) [scheme](https://github.com/google/mediapipe/tree/master/mediapipe/modules/pose_landmark/pose_landmark_full_body_topology.svg). (GPU input, and inference is executed on GPU)
+[`PoseLandmarkCpu`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/pose_landmark/pose_landmark_cpu.pbtxt)| Detects landmarks of a single body pose, full-body by default but can be configured (via an input side packet) to cover upper-body only. See landmarks (aka keypoints) [scheme](https://github.com/google/mediapipe/tree/master/mediapipe/modules/pose_landmark/pose_landmark_full_body_topology.svg). (CPU input, and inference is executed on CPU)
+[`PoseLandmarkGpu`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/pose_landmark/pose_landmark_gpu.pbtxt)| Detects landmarks of a single body pose, full-body by default but can be configured (via an input side packet) to cover upper-body only. See landmarks (aka keypoints) [scheme](https://github.com/google/mediapipe/tree/master/mediapipe/modules/pose_landmark/pose_landmark_full_body_topology.svg). (GPU input, and inference is executed on GPU.)
+[`PoseLandmarkUpperBodyCpu`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/pose_landmark/pose_landmark_upper_body_cpu.pbtxt)| Detects and tracks landmarks of a single upper-body pose. See landmarks (aka keypoints) [scheme](https://github.com/google/mediapipe/tree/master/mediapipe/modules/pose_landmark/pose_landmark_upper_body_topology.svg). (CPU input, and inference is executed on CPU)
+[`PoseLandmarkUpperBodyGpu`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/pose_landmark/pose_landmark_upper_body_gpu.pbtxt)| Detects and tracks landmarks of a single upper-body pose. See landmarks (aka keypoints) [scheme](https://github.com/google/mediapipe/tree/master/mediapipe/modules/pose_landmark/pose_landmark_upper_body_topology.svg). (GPU input, and inference is executed on GPU.)

mediapipe/modules/pose_landmark/pose_detection_to_roi.pbtxt

@@ -9,6 +9,10 @@ type: "PoseDetectionToRoi"
 input_stream: "DETECTION:detection"
 # Frame size (width and height). (std::pair<int, int>)
 input_stream: "IMAGE_SIZE:image_size"
+# Whether to detect/predict the full set of pose landmarks, or only those on the
+# upper body. If unspecified, functions as set to false. (bool)
+input_side_packet: "UPPER_BODY_ONLY:upper_body_only"
+
 # ROI according to the first detection of input detections. (NormalizedRect)
 output_stream: "ROI:roi"
 
@@ -16,16 +20,33 @@ output_stream: "ROI:roi"
 # points. Pose detection contains four key points: first two for full-body pose
 # and two more for upper-body pose.
 node {
-  calculator: "AlignmentPointsRectsCalculator"
+  calculator: "SwitchContainer"
+  input_side_packet: "ENABLE:upper_body_only"
   input_stream: "DETECTION:detection"
   input_stream: "IMAGE_SIZE:image_size"
   output_stream: "NORM_RECT:raw_roi"
-  options: {
-    [mediapipe.DetectionsToRectsCalculatorOptions.ext] {
-      rotation_vector_start_keypoint_index: 2
-      rotation_vector_end_keypoint_index: 3
-      rotation_vector_target_angle_degrees: 90
-      output_zero_rect_for_empty_detections: true
+  options {
+    [mediapipe.SwitchContainerOptions.ext] {
+      contained_node: {
+        calculator: "AlignmentPointsRectsCalculator"
+        options: {
+          [mediapipe.DetectionsToRectsCalculatorOptions.ext] {
+            rotation_vector_start_keypoint_index: 0
+            rotation_vector_end_keypoint_index: 1
+            rotation_vector_target_angle_degrees: 90
+          }
+        }
+      }
+      contained_node: {
+        calculator: "AlignmentPointsRectsCalculator"
+        options: {
+          [mediapipe.DetectionsToRectsCalculatorOptions.ext] {
+            rotation_vector_start_keypoint_index: 2
+            rotation_vector_end_keypoint_index: 3
+            rotation_vector_target_angle_degrees: 90
+          }
+        }
+      }
     }
   }
 }

mediapipe/modules/pose_landmark/pose_landmark_by_roi_cpu.pbtxt

@@ -1,19 +1,24 @@
 # MediaPipe graph to detect/predict upper-body pose landmarks. (CPU input, and
 # inference is executed on CPU.)
 #
-# It is required that "pose_landmark_upper_body.tflite" is available at
+# It is required that "pose_landmark_full_body.tflite" or
+# "pose_landmark_upper_body.tflite" is available at
+# "mediapipe/modules/pose_landmark/pose_landmark_full_body.tflite"
+# or
 # "mediapipe/modules/pose_landmark/pose_landmark_upper_body.tflite"
-# path during execution.
+# path respectively during execution, depending on the specification in the
+# UPPER_BODY_ONLY input side packet.
 #
 # EXAMPLE:
 #   node {
-#     calculator: "PoseLandmarkUpperBodyByRoiCpu"
+#     calculator: "PoseLandmarkByRoiCpu"
+#     input_side_packet: "UPPER_BODY_ONLY:upper_body_only"
 #     input_stream: "IMAGE:image"
 #     input_stream: "ROI:roi"
 #     output_stream: "LANDMARKS:landmarks"
 #   }
 
-type: "PoseLandmarkUpperBodyByRoiCpu"
+type: "PoseLandmarkByRoiCpu"
 
 # CPU image. (ImageFrame)
 input_stream: "IMAGE:image"
@@ -21,33 +26,49 @@ input_stream: "IMAGE:image"
 # (NormalizedRect)
 input_stream: "ROI:roi"
 
+# Whether to detect/predict the full set of pose landmarks (see below), or only
+# those on the upper body. If unspecified, functions as set to false. (bool)
+# Note that upper-body-only prediction may be more accurate for use cases where
+# the lower-body parts are mostly out of view.
+input_side_packet: "UPPER_BODY_ONLY:upper_body_only"
+
 # Pose landmarks within the given ROI. (NormalizedLandmarkList)
-# We have 25 (upper-body) landmarks (see pose_landmark_upper_body_topology.svg).
+# We have 33 landmarks (see pose_landmark_full_body_topology.svg) with the
+# first 25 fall on the upper body (see pose_landmark_upper_body_topology.svg),
+# and there are other auxiliary key points.
 # 0 - nose
-# 1 - right eye (inner)
-# 2 - right eye
-# 3 - right eye (outer)
-# 4 - left eye (inner)
-# 5 - left eye
-# 6 - left eye (outer)
-# 7 - right ear
-# 8 - left ear
-# 9 - mouth (right)
-# 10 - mouth (left)
-# 11 - right shoulder
-# 12 - left shoulder
-# 13 - right elbow
-# 14 - left elbow
-# 15 - right wrist
-# 16 - left wrist
-# 17 - right pinky
-# 18 - left pinky
-# 19 - right index
-# 20 - left index
-# 21 - right thumb
-# 22 - left thumb
-# 23 - right hip
-# 24 - left hip
+# 1 - left eye (inner)
+# 2 - left eye
+# 3 - left eye (outer)
+# 4 - right eye (inner)
+# 5 - right eye
+# 6 - right eye (outer)
+# 7 - left ear
+# 8 - right ear
+# 9 - mouth (left)
+# 10 - mouth (right)
+# 11 - left shoulder
+# 12 - right shoulder
+# 13 - left elbow
+# 14 - right elbow
+# 15 - left wrist
+# 16 - right wrist
+# 17 - left pinky
+# 18 - right pinky
+# 19 - left index
+# 20 - right index
+# 21 - left thumb
+# 22 - right thumb
+# 23 - left hip
+# 24 - right hip
+# 25 - left knee
+# 26 - right knee
+# 27 - left ankle
+# 28 - right ankle
+# 29 - left heel
+# 30 - right heel
+# 31 - left foot index
+# 32 - right foot index
 #
 # NOTE: if a pose is not present within the given ROI, for this particular
 # timestamp there will not be an output packet in the LANDMARKS stream. However,
@@ -80,14 +101,21 @@ node: {
   }
 }
 
-# Runs a TensorFlow Lite model inference on CPU.
+# Loads the pose landmark TF Lite model.
+node {
+  calculator: "PoseLandmarkModelLoader"
+  input_side_packet: "UPPER_BODY_ONLY:upper_body_only"
+  output_side_packet: "MODEL:model"
+}
+
+# Runs model inference on CPU.
 node {
   calculator: "InferenceCalculator"
+  input_side_packet: "MODEL:model"
   input_stream: "TENSORS:input_tensors"
   output_stream: "TENSORS:output_tensors"
   options: {
     [mediapipe.InferenceCalculatorOptions.ext] {
-      model_path: "mediapipe/modules/pose_landmark/pose_landmark_upper_body.tflite"
       delegate { xnnpack {} }
     }
   }
@@ -140,14 +168,36 @@ node {
 # Decodes the landmark tensors into a vector of landmarks, where the landmark
 # coordinates are normalized by the size of the input image to the model.
 node {
-  calculator: "TensorsToLandmarksCalculator"
+  calculator: "SwitchContainer"
+  input_side_packet: "ENABLE:upper_body_only"
   input_stream: "TENSORS:ensured_landmark_tensors"
   output_stream: "NORM_LANDMARKS:raw_landmarks"
   options: {
-    [mediapipe.TensorsToLandmarksCalculatorOptions.ext] {
-      num_landmarks: 27
-      input_image_width: 256
-      input_image_height: 256
+    [mediapipe.SwitchContainerOptions.ext] {
+      contained_node: {
+        calculator: "TensorsToLandmarksCalculator"
+        options: {
+          [mediapipe.TensorsToLandmarksCalculatorOptions.ext] {
+            num_landmarks: 35
+            input_image_width: 256
+            input_image_height: 256
+            visibility_activation: SIGMOID
+            presence_activation: SIGMOID
+          }
+        }
+      }
+      contained_node: {
+        calculator: "TensorsToLandmarksCalculator"
+        options: {
+          [mediapipe.TensorsToLandmarksCalculatorOptions.ext] {
+            num_landmarks: 27
+            input_image_width: 256
+            input_image_height: 256
+            visibility_activation: SIGMOID
+            presence_activation: SIGMOID
+          }
+        }
+      }
     }
   }
 }
@@ -175,14 +225,31 @@ node {
 # Splits the landmarks into two sets: the actual pose landmarks and the
 # auxiliary landmarks.
 node {
-  calculator: "SplitNormalizedLandmarkListCalculator"
+  calculator: "SwitchContainer"
+  input_side_packet: "ENABLE:upper_body_only"
   input_stream: "all_landmarks"
   output_stream: "landmarks"
   output_stream: "auxiliary_landmarks"
   options: {
-    [mediapipe.SplitVectorCalculatorOptions.ext] {
-      ranges: { begin: 0 end: 25 }
-      ranges: { begin: 25 end: 27 }
+    [mediapipe.SwitchContainerOptions.ext] {
+      contained_node: {
+        calculator: "SplitNormalizedLandmarkListCalculator"
+        options: {
+          [mediapipe.SplitVectorCalculatorOptions.ext] {
+            ranges: { begin: 0 end: 33 }
+            ranges: { begin: 33 end: 35 }
+          }
+        }
+      }
+      contained_node: {
+        calculator: "SplitNormalizedLandmarkListCalculator"
+        options: {
+          [mediapipe.SplitVectorCalculatorOptions.ext] {
+            ranges: { begin: 0 end: 25 }
+            ranges: { begin: 25 end: 27 }
+          }
+        }
+      }
     }
   }
 }

mediapipe/modules/pose_landmark/pose_landmark_by_roi_gpu.pbtxt

@@ -1,19 +1,24 @@
 # MediaPipe graph to detect/predict upper-body pose landmarks. (GPU input, and
 # inference is executed on GPU.)
 #
-# It is required that "pose_landmark_upper_body.tflite" is available at
+# It is required that "pose_landmark_full_body.tflite" or
+# "pose_landmark_upper_body.tflite" is available at
+# "mediapipe/modules/pose_landmark/pose_landmark_full_body.tflite"
+# or
 # "mediapipe/modules/pose_landmark/pose_landmark_upper_body.tflite"
-# path during execution.
+# path respectively during execution, depending on the specification in the
+# UPPER_BODY_ONLY input side packet.
 #
 # EXAMPLE:
 #   node {
-#     calculator: "PoseLandmarkUpperBodyByRoiGpu"
+#     calculator: "PoseLandmarkByRoiGpu"
+#     input_side_packet: "UPPER_BODY_ONLY:upper_body_only"
 #     input_stream: "IMAGE:image"
 #     input_stream: "ROI:roi"
 #     output_stream: "LANDMARKS:landmarks"
 #   }
 
-type: "PoseLandmarkUpperBodyByRoiGpu"
+type: "PoseLandmarkByRoiGpu"
 
 # GPU image. (GpuBuffer)
 input_stream: "IMAGE:image"
@@ -21,33 +26,49 @@ input_stream: "IMAGE:image"
 # (NormalizedRect)
 input_stream: "ROI:roi"
 
+# Whether to detect/predict the full set of pose landmarks (see below), or only
+# those on the upper body. If unspecified, functions as set to false. (bool)
+# Note that upper-body-only prediction may be more accurate for use cases where
+# the lower-body parts are mostly out of view.
+input_side_packet: "UPPER_BODY_ONLY:upper_body_only"
+
 # Pose landmarks within the given ROI. (NormalizedLandmarkList)
-# We have 25 (upper-body) landmarks (see pose_landmark_upper_body_topology.svg).
+# We have 33 landmarks (see pose_landmark_full_body_topology.svg) with the
+# first 25 fall on the upper body (see pose_landmark_upper_body_topology.svg),
+# and there are other auxiliary key points.
 # 0 - nose
-# 1 - right eye (inner)
-# 2 - right eye
-# 3 - right eye (outer)
-# 4 - left eye (inner)
-# 5 - left eye
-# 6 - left eye (outer)
-# 7 - right ear
-# 8 - left ear
-# 9 - mouth (right)
-# 10 - mouth (left)
-# 11 - right shoulder
-# 12 - left shoulder
-# 13 - right elbow
-# 14 - left elbow
-# 15 - right wrist
-# 16 - left wrist
-# 17 - right pinky
-# 18 - left pinky
-# 19 - right index
-# 20 - left index
-# 21 - right thumb
-# 22 - left thumb
-# 23 - right hip
-# 24 - left hip
+# 1 - left eye (inner)
+# 2 - left eye
+# 3 - left eye (outer)
+# 4 - right eye (inner)
+# 5 - right eye
+# 6 - right eye (outer)
+# 7 - left ear
+# 8 - right ear
+# 9 - mouth (left)
+# 10 - mouth (right)
+# 11 - left shoulder
+# 12 - right shoulder
+# 13 - left elbow
+# 14 - right elbow
+# 15 - left wrist
+# 16 - right wrist
+# 17 - left pinky
+# 18 - right pinky
+# 19 - left index
+# 20 - right index
+# 21 - left thumb
+# 22 - right thumb
+# 23 - left hip
+# 24 - right hip
+# 25 - left knee
+# 26 - right knee
+# 27 - left ankle
+# 28 - right ankle
+# 29 - left heel
+# 30 - right heel
+# 31 - left foot index
+# 32 - right foot index
 #
 # NOTE: if a pose is not present within the given ROI, for this particular
 # timestamp there will not be an output packet in the LANDMARKS stream. However,
@@ -81,16 +102,19 @@ node: {
   }
 }
 
-# Runs a TensorFlow Lite model inference on GPU.
+# Loads the pose landmark TF Lite model.
+node {
+  calculator: "PoseLandmarkModelLoader"
+  input_side_packet: "UPPER_BODY_ONLY:upper_body_only"
+  output_side_packet: "MODEL:model"
+}
+
+# Runs model inference on GPU.
 node {
   calculator: "InferenceCalculator"
+  input_side_packet: "MODEL:model"
   input_stream: "TENSORS:input_tensors"
   output_stream: "TENSORS:output_tensors"
-  options: {
-    [mediapipe.InferenceCalculatorOptions.ext] {
-      model_path: "mediapipe/modules/pose_landmark/pose_landmark_upper_body.tflite"
-    }
-  }
 }
 
 # Splits a vector of TFLite tensors to multiple vectors according to the ranges
@@ -140,14 +164,36 @@ node {
 # Decodes the landmark tensors into a vector of landmarks, where the landmark
 # coordinates are normalized by the size of the input image to the model.
 node {
-  calculator: "TensorsToLandmarksCalculator"
+  calculator: "SwitchContainer"
+  input_side_packet: "ENABLE:upper_body_only"
   input_stream: "TENSORS:ensured_landmark_tensors"
   output_stream: "NORM_LANDMARKS:raw_landmarks"
   options: {
-    [mediapipe.TensorsToLandmarksCalculatorOptions.ext] {
-      num_landmarks: 27
-      input_image_width: 256
-      input_image_height: 256
+    [mediapipe.SwitchContainerOptions.ext] {
+      contained_node: {
+        calculator: "TensorsToLandmarksCalculator"
+        options: {
+          [mediapipe.TensorsToLandmarksCalculatorOptions.ext] {
+            num_landmarks: 35
+            input_image_width: 256
+            input_image_height: 256
+            visibility_activation: SIGMOID
+            presence_activation: SIGMOID
+          }
+        }
+      }
+      contained_node: {
+        calculator: "TensorsToLandmarksCalculator"
+        options: {
+          [mediapipe.TensorsToLandmarksCalculatorOptions.ext] {
+            num_landmarks: 27
+            input_image_width: 256
+            input_image_height: 256
+            visibility_activation: SIGMOID
+            presence_activation: SIGMOID
+          }
+        }
+      }
     }
   }
 }
@@ -175,14 +221,31 @@ node {
 # Splits the landmarks into two sets: the actual pose landmarks and the
 # auxiliary landmarks.
 node {
-  calculator: "SplitNormalizedLandmarkListCalculator"
+  calculator: "SwitchContainer"
+  input_side_packet: "ENABLE:upper_body_only"
   input_stream: "all_landmarks"
   output_stream: "landmarks"
   output_stream: "auxiliary_landmarks"
   options: {
-    [mediapipe.SplitVectorCalculatorOptions.ext] {
-      ranges: { begin: 0 end: 25 }
-      ranges: { begin: 25 end: 27 }
+    [mediapipe.SwitchContainerOptions.ext] {
+      contained_node: {
+        calculator: "SplitNormalizedLandmarkListCalculator"
+        options: {
+          [mediapipe.SplitVectorCalculatorOptions.ext] {
+            ranges: { begin: 0 end: 33 }
+            ranges: { begin: 33 end: 35 }
+          }
+        }
+      }
+      contained_node: {
+        calculator: "SplitNormalizedLandmarkListCalculator"
+        options: {
+          [mediapipe.SplitVectorCalculatorOptions.ext] {
+            ranges: { begin: 0 end: 25 }
+            ranges: { begin: 25 end: 27 }
+          }
+        }
+      }
     }
   }
 }

mediapipe/modules/pose_landmark/pose_landmark_cpu.pbtxt

@@ -0,0 +1,227 @@
+# MediaPipe graph to detect/predict pose landmarks. (CPU input, and inference is
+# executed on CPU.) This graph tries to skip pose detection as much as possible
+# by using previously detected/predicted landmarks for new images.
+#
+# It is required that "pose_detection.tflite" is available at
+# "mediapipe/modules/pose_detection/pose_detection.tflite"
+# path during execution.
+#
+# It is required that "pose_landmark_full_body.tflite" or
+# "pose_landmark_upper_body.tflite" is available at
+# "mediapipe/modules/pose_landmark/pose_landmark_full_body.tflite"
+# or
+# "mediapipe/modules/pose_landmark/pose_landmark_upper_body.tflite"
+# path respectively during execution, depending on the specification in the
+# UPPER_BODY_ONLY input side packet.
+#
+# EXAMPLE:
+#   node {
+#     calculator: "PoseLandmarkCpu"
+#     input_side_packet: "UPPER_BODY_ONLY:upper_body_only"
+#     input_side_packet: "SMOOTH_LANDMARKS:smooth_landmarks"
+#     input_stream: "IMAGE:image"
+#     output_stream: "LANDMARKS:pose_landmarks"
+#   }
+
+type: "PoseLandmarkCpu"
+
+# CPU image. (ImageFrame)
+input_stream: "IMAGE:image"
+
+# Whether to detect/predict the full set of pose landmarks (see below), or only
+# those on the upper body. If unspecified, functions as set to false. (bool)
+# Note that upper-body-only prediction may be more accurate for use cases where
+# the lower-body parts are mostly out of view.
+input_side_packet: "UPPER_BODY_ONLY:upper_body_only"
+
+# Whether to filter landmarks across different input images to reduce jitter.
+# If unspecified, functions as set to false. (bool)
+input_side_packet: "SMOOTH_LANDMARKS:smooth_landmarks"
+
+# Pose landmarks within the given ROI. (NormalizedLandmarkList)
+# We have 33 landmarks (see pose_landmark_full_body_topology.svg) with the
+# first 25 fall on the upper body (see pose_landmark_upper_body_topology.svg),
+# and there are other auxiliary key points.
+# 0 - nose
+# 1 - left eye (inner)
+# 2 - left eye
+# 3 - left eye (outer)
+# 4 - right eye (inner)
+# 5 - right eye
+# 6 - right eye (outer)
+# 7 - left ear
+# 8 - right ear
+# 9 - mouth (left)
+# 10 - mouth (right)
+# 11 - left shoulder
+# 12 - right shoulder
+# 13 - left elbow
+# 14 - right elbow
+# 15 - left wrist
+# 16 - right wrist
+# 17 - left pinky
+# 18 - right pinky
+# 19 - left index
+# 20 - right index
+# 21 - left thumb
+# 22 - right thumb
+# 23 - left hip
+# 24 - right hip
+# 25 - left knee
+# 26 - right knee
+# 27 - left ankle
+# 28 - right ankle
+# 29 - left heel
+# 30 - right heel
+# 31 - left foot index
+# 32 - right foot index
+#
+# NOTE: if a pose is not present within the given ROI, for this particular
+# timestamp there will not be an output packet in the LANDMARKS stream. However,
+# the MediaPipe framework will internally inform the downstream calculators of
+# the absence of this packet so that they don't wait for it unnecessarily.
+output_stream: "LANDMARKS:pose_landmarks"
+
+# Extra outputs (for debugging, for instance).
+# Detected poses. (Detection)
+output_stream: "DETECTION:pose_detection"
+# Regions of interest calculated based on landmarks. (NormalizedRect)
+output_stream: "ROI_FROM_LANDMARKS:pose_rect_from_landmarks"
+# Regions of interest calculated based on pose detections. (NormalizedRect)
+output_stream: "ROI_FROM_DETECTION:pose_rect_from_detection"
+
+# Defines whether landmarks on the previous image should be used to help
+# localize landmarks on the current image.
+node {
+  name: "ConstantSidePacketCalculator"
+  calculator: "ConstantSidePacketCalculator"
+  output_side_packet: "PACKET:use_prev_landmarks"
+  options: {
+    [mediapipe.ConstantSidePacketCalculatorOptions.ext]: {
+      packet { bool_value: true }
+    }
+  }
+}
+node {
+  calculator: "GateCalculator"
+  input_side_packet: "ALLOW:use_prev_landmarks"
+  input_stream: "prev_pose_rect_from_landmarks"
+  output_stream: "gated_prev_pose_rect_from_landmarks"
+}
+
+# Checks if there's previous pose rect calculated from landmarks.
+node: {
+  calculator: "PacketPresenceCalculator"
+  input_stream: "PACKET:gated_prev_pose_rect_from_landmarks"
+  output_stream: "PRESENCE:prev_pose_rect_from_landmarks_is_present"
+}
+
+# Calculates size of the image.
+node {
+  calculator: "ImagePropertiesCalculator"
+  input_stream: "IMAGE:image"
+  output_stream: "SIZE:image_size"
+}
+
+# Drops the incoming image if the pose has already been identified from the
+# previous image. Otherwise, passes the incoming image through to trigger a new
+# round of pose detection.
+node {
+  calculator: "GateCalculator"
+  input_stream: "image"
+  input_stream: "image_size"
+  input_stream: "DISALLOW:prev_pose_rect_from_landmarks_is_present"
+  output_stream: "image_for_pose_detection"
+  output_stream: "image_size_for_pose_detection"
+  options: {
+    [mediapipe.GateCalculatorOptions.ext] {
+      empty_packets_as_allow: true
+    }
+  }
+}
+
+# Detects poses.
+node {
+  calculator: "PoseDetectionCpu"
+  input_stream: "IMAGE:image_for_pose_detection"
+  output_stream: "DETECTIONS:pose_detections"
+}
+
+# Gets the very first detection from "pose_detections" vector.
+node {
+  calculator: "SplitDetectionVectorCalculator"
+  input_stream: "pose_detections"
+  output_stream: "pose_detection"
+  options: {
+    [mediapipe.SplitVectorCalculatorOptions.ext] {
+      ranges: { begin: 0 end: 1 }
+      element_only: true
+    }
+  }
+}
+
+# Calculates region of interest based on pose detection, so that can be used
+# to detect landmarks.
+node {
+  calculator: "PoseDetectionToRoi"
+  input_side_packet: "UPPER_BODY_ONLY:upper_body_only"
+  input_stream: "DETECTION:pose_detection"
+  input_stream: "IMAGE_SIZE:image_size_for_pose_detection"
+  output_stream: "ROI:pose_rect_from_detection"
+}
+
+# Selects either pose rect (or ROI) calculated from detection or from previously
+# detected landmarks if available (in this case, calculation of pose rect from
+# detection is skipped).
+node {
+  calculator: "MergeCalculator"
+  input_stream: "pose_rect_from_detection"
+  input_stream: "gated_prev_pose_rect_from_landmarks"
+  output_stream: "pose_rect"
+}
+
+# Detects pose landmarks within specified region of interest of the image.
+node {
+  calculator: "PoseLandmarkByRoiCpu"
+  input_side_packet: "UPPER_BODY_ONLY:upper_body_only"
+  input_stream: "IMAGE:image"
+  input_stream: "ROI:pose_rect"
+  output_stream: "LANDMARKS:unfiltered_pose_landmarks"
+  output_stream: "AUXILIARY_LANDMARKS:unfiltered_auxiliary_landmarks"
+}
+
+# Smoothes landmarks to reduce jitter.
+node {
+  calculator: "PoseLandmarkFiltering"
+  input_side_packet: "ENABLE:smooth_landmarks"
+  input_stream: "IMAGE_SIZE:image_size"
+  input_stream: "NORM_LANDMARKS:unfiltered_pose_landmarks"
+  input_stream: "AUX_NORM_LANDMARKS:unfiltered_auxiliary_landmarks"
+  output_stream: "FILTERED_NORM_LANDMARKS:pose_landmarks"
+  output_stream: "FILTERED_AUX_NORM_LANDMARKS:auxiliary_landmarks"
+}
+
+# Calculates region of interest based on the auxiliary landmarks, to be used in
+# the subsequent image.
+node {
+  calculator: "PoseLandmarksToRoi"
+  input_stream: "LANDMARKS:auxiliary_landmarks"
+  input_stream: "IMAGE_SIZE:image_size"
+  output_stream: "ROI:pose_rect_from_landmarks"
+}
+
+# Caches pose rects calculated from landmarks, and upon the arrival of the next
+# input image, sends out the cached rects with timestamps replaced by that of
+# the input image, essentially generating a packet that carries the previous
+# pose rects. Note that upon the arrival of the very first input image, a
+# timestamp bound update occurs to jump start the feedback loop.
+node {
+  calculator: "PreviousLoopbackCalculator"
+  input_stream: "MAIN:image"
+  input_stream: "LOOP:pose_rect_from_landmarks"
+  input_stream_info: {
+    tag_index: "LOOP"
+    back_edge: true
+  }
+  output_stream: "PREV_LOOP:prev_pose_rect_from_landmarks"
+}

mediapipe/modules/pose_landmark/pose_landmark_filtering.pbtxt

@@ -0,0 +1,94 @@
+# MediaPipe graph to filter landmarks temporally (across packets with
+# incremental timestamps) to reduce jitter.
+#
+# EXAMPLE:
+#   node {
+#     calculator: "PoseLandmarkFiltering"
+#     input_side_packet: "ENABLE:enable"
+#     input_stream: "IMAGE_SIZE:image_size"
+#     input_stream: "NORM_LANDMARKS:landmarks"
+#     input_stream: "AUX_NORM_LANDMARKS:aux_landmarks"
+#     output_stream: "FILTERED_NORM_LANDMARKS:filtered_landmarks"
+#     output_stream: "FILTERED_AUX_NORM_LANDMARKS:filtered_aux_landmarks"
+#   }
+
+type: "PoseLandmarkFiltering"
+
+# Whether to enable filtering. If unspecified, functions as not enabled. (bool)
+input_side_packet: "ENABLE:enable"
+
+# Size of the image (width & height) where the landmarks are estimated from.
+# (std::pair<int, int>)
+input_stream: "IMAGE_SIZE:image_size"
+# Normalized landmarks. (NormalizedRect)
+input_stream: "NORM_LANDMARKS:landmarks"
+# Auxiliary set of normalized landmarks. (NormalizedRect)
+input_stream: "AUX_NORM_LANDMARKS:aux_landmarks"
+# Filtered normalized landmarks. (NormalizedRect)
+output_stream: "FILTERED_NORM_LANDMARKS:filtered_landmarks"
+# Filtered auxiliary set of normalized landmarks. (NormalizedRect)
+output_stream: "FILTERED_AUX_NORM_LANDMARKS:filtered_aux_landmarks"
+
+# Smoothes pose landmarks to reduce jitter.
+node {
+  calculator: "SwitchContainer"
+  input_side_packet: "ENABLE:enable"
+  input_stream: "NORM_LANDMARKS:landmarks"
+  input_stream: "IMAGE_SIZE:image_size"
+  output_stream: "NORM_FILTERED_LANDMARKS:filtered_landmarks"
+  options: {
+    [mediapipe.SwitchContainerOptions.ext] {
+      contained_node: {
+        calculator: "LandmarksSmoothingCalculator"
+        options: {
+          [mediapipe.LandmarksSmoothingCalculatorOptions.ext] {
+            no_filter: {}
+          }
+        }
+      }
+      contained_node: {
+        calculator: "LandmarksSmoothingCalculator"
+        options: {
+          [mediapipe.LandmarksSmoothingCalculatorOptions.ext] {
+            velocity_filter: {
+              window_size: 5
+              velocity_scale: 10.0
+            }
+          }
+        }
+      }
+    }
+  }
+}
+
+# Smoothes auxiliary landmarks to reduce jitter.
+node {
+  calculator: "SwitchContainer"
+  input_side_packet: "ENABLE:enable"
+  input_stream: "NORM_LANDMARKS:aux_landmarks"
+  input_stream: "IMAGE_SIZE:image_size"
+  output_stream: "NORM_FILTERED_LANDMARKS:filtered_aux_landmarks"
+  options: {
+    [mediapipe.SwitchContainerOptions.ext] {
+      contained_node: {
+        calculator: "LandmarksSmoothingCalculator"
+        options: {
+          [mediapipe.LandmarksSmoothingCalculatorOptions.ext] {
+            no_filter: {}
+          }
+        }
+      }
+      contained_node: {
+        calculator: "LandmarksSmoothingCalculator"
+        options: {
+          [mediapipe.LandmarksSmoothingCalculatorOptions.ext] {
+            velocity_filter: {
+              window_size: 5
+              velocity_scale: 10.0
+            }
+          }
+        }
+      }
+    }
+  }
+}