Project import generated by Copybara.

GitOrigin-RevId: d8caa66de45839696f5bd0786ad3bfbcb9cff632

mediapipe/graphs/face_mesh/calculators/face_landmarks_to_render_data_calculator.cc

@@ -81,11 +81,11 @@ constexpr int kFaceLandmarkConnections[] = {
 class FaceLandmarksToRenderDataCalculator
     : public LandmarksToRenderDataCalculator {
  public:
-  ::mediapipe::Status Open(CalculatorContext* cc) override;
+  mediapipe::Status Open(CalculatorContext* cc) override;
 };
 REGISTER_CALCULATOR(FaceLandmarksToRenderDataCalculator);
 
-::mediapipe::Status FaceLandmarksToRenderDataCalculator::Open(
+mediapipe::Status FaceLandmarksToRenderDataCalculator::Open(
     CalculatorContext* cc) {
   cc->SetOffset(TimestampDiff(0));
   options_ = cc->Options<mediapipe::LandmarksToRenderDataCalculatorOptions>();
@@ -95,7 +95,7 @@ REGISTER_CALCULATOR(FaceLandmarksToRenderDataCalculator);
     landmark_connections_.push_back(kFaceLandmarkConnections[i * 2 + 1]);
   }
 
-  return ::mediapipe::OkStatus();
+  return mediapipe::OkStatus();
 }
 
 }  // namespace mediapipe

mediapipe/graphs/holistic_tracking/BUILD

@@ -0,0 +1,73 @@
+# Copyright 2020 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load(
+    "//mediapipe/framework/tool:mediapipe_graph.bzl",
+    "mediapipe_binary_graph",
+    "mediapipe_simple_subgraph",
+)
+
+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])
+
+mediapipe_simple_subgraph(
+    name = "holistic_tracking_to_render_data",
+    graph = "holistic_tracking_to_render_data.pbtxt",
+    register_as = "HolisticTrackingToRenderData",
+    deps = [
+        "//mediapipe/calculators/core:concatenate_normalized_landmark_list_calculator",
+        "//mediapipe/calculators/core:concatenate_vector_calculator",
+        "//mediapipe/calculators/core:merge_calculator",
+        "//mediapipe/calculators/core:split_normalized_landmark_list_calculator",
+        "//mediapipe/calculators/core:split_vector_calculator",
+        "//mediapipe/calculators/util:detections_to_render_data_calculator",
+        "//mediapipe/calculators/util:landmarks_to_render_data_calculator",
+        "//mediapipe/calculators/util:rect_to_render_data_calculator",
+        "//mediapipe/calculators/util:rect_to_render_scale_calculator",
+        "//mediapipe/framework/tool:switch_container",
+        "//mediapipe/modules/holistic_landmark:hand_wrist_for_pose",
+    ],
+)
+
+cc_library(
+    name = "holistic_tracking_gpu_deps",
+    deps = [
+        ":holistic_tracking_to_render_data",
+        "//mediapipe/calculators/core:constant_side_packet_calculator",
+        "//mediapipe/calculators/core:flow_limiter_calculator",
+        "//mediapipe/calculators/image:image_properties_calculator",
+        "//mediapipe/calculators/util:annotation_overlay_calculator",
+        "//mediapipe/modules/holistic_landmark:holistic_landmark_gpu",
+    ],
+)
+
+mediapipe_binary_graph(
+    name = "holistic_tracking_gpu",
+    graph = "holistic_tracking_gpu.pbtxt",
+    output_name = "holistic_tracking_gpu.binarypb",
+    deps = [":holistic_tracking_gpu_deps"],
+)
+
+cc_library(
+    name = "holistic_tracking_cpu_graph_deps",
+    deps = [
+        ":holistic_tracking_to_render_data",
+        "//mediapipe/calculators/core:constant_side_packet_calculator",
+        "//mediapipe/calculators/core:flow_limiter_calculator",
+        "//mediapipe/calculators/image:image_properties_calculator",
+        "//mediapipe/calculators/util:annotation_overlay_calculator",
+        "//mediapipe/modules/holistic_landmark:holistic_landmark_cpu",
+    ],
+)

mediapipe/graphs/holistic_tracking/holistic_tracking_cpu.pbtxt

@@ -0,0 +1,90 @@
+# Tracks and renders pose + hands + face landmarks.
+
+# CPU image. (ImageFrame)
+input_stream: "input_video"
+
+# CPU image with rendered results. (ImageFrame)
+output_stream: "output_video"
+
+# Throttles the images flowing downstream for flow control. It passes through
+# the very first incoming image unaltered, and waits for downstream nodes
+# (calculators and subgraphs) in the graph to finish their tasks before it
+# passes through another image. All images that come in while waiting are
+# dropped, limiting the number of in-flight images in most part of the graph to
+# 1. This prevents the downstream nodes from queuing up incoming images and data
+# excessively, which leads to increased latency and memory usage, unwanted in
+# real-time mobile applications. It also eliminates unnecessarily computation,
+# e.g., the output produced by a node may get dropped downstream if the
+# subsequent nodes are still busy processing previous inputs.
+node {
+  calculator: "FlowLimiterCalculator"
+  input_stream: "input_video"
+  input_stream: "FINISHED:output_video"
+  input_stream_info: {
+    tag_index: "FINISHED"
+    back_edge: true
+  }
+  output_stream: "throttled_input_video"
+  node_options: {
+    [type.googleapis.com/mediapipe.FlowLimiterCalculatorOptions] {
+      max_in_flight: 1
+      max_in_queue: 1
+      # Timeout is disabled (set to 0) as first frame processing can take more
+      # than 1 second.
+      in_flight_timeout: 0
+    }
+  }
+}
+
+node {
+  calculator: "ConstantSidePacketCalculator"
+  output_side_packet: "PACKET:0:upper_body_only"
+  output_side_packet: "PACKET:1:smooth_landmarks"
+  node_options: {
+    [type.googleapis.com/mediapipe.ConstantSidePacketCalculatorOptions]: {
+      packet { bool_value: false }
+      packet { bool_value: true }
+    }
+  }
+}
+
+node {
+  calculator: "HolisticLandmarkCpu"
+  input_stream: "IMAGE:throttled_input_video"
+  input_side_packet: "UPPER_BODY_ONLY:upper_body_only"
+  input_side_packet: "SMOOTH_LANDMARKS:smooth_landmarks"
+  output_stream: "POSE_LANDMARKS:pose_landmarks"
+  output_stream: "POSE_ROI:pose_roi"
+  output_stream: "POSE_DETECTION:pose_detection"
+  output_stream: "FACE_LANDMARKS:face_landmarks"
+  output_stream: "LEFT_HAND_LANDMARKS:left_hand_landmarks"
+  output_stream: "RIGHT_HAND_LANDMARKS:right_hand_landmarks"
+}
+
+# Gets image size.
+node {
+  calculator: "ImagePropertiesCalculator"
+  input_stream: "IMAGE:throttled_input_video"
+  output_stream: "SIZE:image_size"
+}
+
+# Converts pose, hands and face landmarks to a render data vector.
+node {
+  calculator: "HolisticTrackingToRenderData"
+  input_stream: "IMAGE_SIZE:image_size"
+  input_stream: "POSE_LANDMARKS:pose_landmarks"
+  input_stream: "POSE_ROI:pose_roi"
+  input_stream: "LEFT_HAND_LANDMARKS:left_hand_landmarks"
+  input_stream: "RIGHT_HAND_LANDMARKS:right_hand_landmarks"
+  input_stream: "FACE_LANDMARKS:face_landmarks"
+  input_side_packet: "UPPER_BODY_ONLY:upper_body_only"
+  output_stream: "RENDER_DATA_VECTOR:render_data_vector"
+}
+
+# Draws annotations and overlays them on top of the input images.
+node {
+  calculator: "AnnotationOverlayCalculator"
+  input_stream: "IMAGE:throttled_input_video"
+  input_stream: "VECTOR:render_data_vector"
+  output_stream: "IMAGE:output_video"
+}

mediapipe/graphs/holistic_tracking/holistic_tracking_gpu.pbtxt

@@ -0,0 +1,90 @@
+# Tracks and renders pose + hands + face landmarks.
+
+# GPU buffer. (GpuBuffer)
+input_stream: "input_video"
+
+# GPU image with rendered results. (GpuBuffer)
+output_stream: "output_video"
+
+# Throttles the images flowing downstream for flow control. It passes through
+# the very first incoming image unaltered, and waits for downstream nodes
+# (calculators and subgraphs) in the graph to finish their tasks before it
+# passes through another image. All images that come in while waiting are
+# dropped, limiting the number of in-flight images in most part of the graph to
+# 1. This prevents the downstream nodes from queuing up incoming images and data
+# excessively, which leads to increased latency and memory usage, unwanted in
+# real-time mobile applications. It also eliminates unnecessarily computation,
+# e.g., the output produced by a node may get dropped downstream if the
+# subsequent nodes are still busy processing previous inputs.
+node {
+  calculator: "FlowLimiterCalculator"
+  input_stream: "input_video"
+  input_stream: "FINISHED:output_video"
+  input_stream_info: {
+    tag_index: "FINISHED"
+    back_edge: true
+  }
+  output_stream: "throttled_input_video"
+  node_options: {
+    [type.googleapis.com/mediapipe.FlowLimiterCalculatorOptions] {
+      max_in_flight: 1
+      max_in_queue: 1
+      # Timeout is disabled (set to 0) as first frame processing can take more
+      # than 1 second.
+      in_flight_timeout: 0
+    }
+  }
+}
+
+node {
+  calculator: "ConstantSidePacketCalculator"
+  output_side_packet: "PACKET:0:upper_body_only"
+  output_side_packet: "PACKET:1:smooth_landmarks"
+  node_options: {
+    [type.googleapis.com/mediapipe.ConstantSidePacketCalculatorOptions]: {
+      packet { bool_value: false }
+      packet { bool_value: true }
+    }
+  }
+}
+
+node {
+  calculator: "HolisticLandmarkGpu"
+  input_stream: "IMAGE:throttled_input_video"
+  input_side_packet: "UPPER_BODY_ONLY:upper_body_only"
+  input_side_packet: "SMOOTH_LANDMARKS:smooth_landmarks"
+  output_stream: "POSE_LANDMARKS:pose_landmarks"
+  output_stream: "POSE_ROI:pose_roi"
+  output_stream: "POSE_DETECTION:pose_detection"
+  output_stream: "FACE_LANDMARKS:face_landmarks"
+  output_stream: "LEFT_HAND_LANDMARKS:left_hand_landmarks"
+  output_stream: "RIGHT_HAND_LANDMARKS:right_hand_landmarks"
+}
+
+# Gets image size.
+node {
+  calculator: "ImagePropertiesCalculator"
+  input_stream: "IMAGE_GPU:throttled_input_video"
+  output_stream: "SIZE:image_size"
+}
+
+# Converts pose, hands and face landmarks to a render data vector.
+node {
+  calculator: "HolisticTrackingToRenderData"
+  input_stream: "IMAGE_SIZE:image_size"
+  input_stream: "POSE_LANDMARKS:pose_landmarks"
+  input_stream: "POSE_ROI:pose_roi"
+  input_stream: "LEFT_HAND_LANDMARKS:left_hand_landmarks"
+  input_stream: "RIGHT_HAND_LANDMARKS:right_hand_landmarks"
+  input_stream: "FACE_LANDMARKS:face_landmarks"
+  input_side_packet: "UPPER_BODY_ONLY:upper_body_only"
+  output_stream: "RENDER_DATA_VECTOR:render_data_vector"
+}
+
+# Draws annotations and overlays them on top of the input images.
+node {
+  calculator: "AnnotationOverlayCalculator"
+  input_stream: "IMAGE_GPU:throttled_input_video"
+  input_stream: "VECTOR:render_data_vector"
+  output_stream: "IMAGE_GPU:output_video"
+}

mediapipe/graphs/holistic_tracking/holistic_tracking_to_render_data.pbtxt

@@ -0,0 +1,854 @@
+# Converts pose + hands + face landmarks to a render data vector.
+
+type: "HolisticTrackingToRenderData"
+
+# Image size. (std::pair<int, int>)
+input_stream: "IMAGE_SIZE:image_size"
+# Pose landmarks. (NormalizedLandmarkList)
+input_stream: "POSE_LANDMARKS:landmarks"
+# Region of interest calculated based on pose landmarks. (NormalizedRect)
+input_stream: "POSE_ROI:roi"
+# Left hand landmarks. (NormalizedLandmarkList)
+input_stream: "LEFT_HAND_LANDMARKS:left_hand_landmarks"
+# Right hand landmarks. (NormalizedLandmarkList)
+input_stream: "RIGHT_HAND_LANDMARKS:right_hand_landmarks"
+# Face landmarks. (NormalizedLandmarkList)
+input_stream: "FACE_LANDMARKS:face_landmarks"
+
+# Whether to render the full set of pose landmarks, or only those on the
+# upper body. If unspecified, functions as set to false. (bool)
+input_side_packet: "UPPER_BODY_ONLY:upper_body_only"
+
+# Render data vector. (std::vector<RenderData>)
+output_stream: "RENDER_DATA_VECTOR:render_data_vector"
+
+# --------------------------------------------------------------------------- #
+# ------------------ Calculates scale for render objects -------------------- #
+# --------------------------------------------------------------------------- #
+
+# Calculates rendering scale based on the pose bounding box.
+node {
+  calculator: "RectToRenderScaleCalculator"
+  input_stream: "NORM_RECT:roi"
+  input_stream: "IMAGE_SIZE:image_size"
+  output_stream: "RENDER_SCALE:render_scale"
+  node_options: {
+    [type.googleapis.com/mediapipe.RectToRenderScaleCalculatorOptions] {
+      multiplier: 0.0008
+    }
+  }
+}
+
+# --------------------------------------------------------------------------- #
+# --------------- Combines pose and hands into pose skeleton ---------------- #
+# --------------------------------------------------------------------------- #
+
+# Gets pose landmarks before wrists.
+node {
+  calculator: "SplitNormalizedLandmarkListCalculator"
+  input_stream: "landmarks"
+  output_stream: "landmarks_before_wrist"
+  node_options: {
+    [type.googleapis.com/mediapipe.SplitVectorCalculatorOptions] {
+      ranges: { begin: 11 end: 15 }
+    }
+  }
+}
+
+# Gets pose left wrist landmark.
+node {
+  calculator: "SplitNormalizedLandmarkListCalculator"
+  input_stream: "landmarks"
+  output_stream: "landmarks_left_wrist"
+  node_options: {
+    [type.googleapis.com/mediapipe.SplitVectorCalculatorOptions] {
+      ranges: { begin: 15 end: 16 }
+    }
+  }
+}
+
+# Gets pose right wrist landmark.
+node {
+  calculator: "SplitNormalizedLandmarkListCalculator"
+  input_stream: "landmarks"
+  output_stream: "landmarks_right_wrist"
+  node_options: {
+    [type.googleapis.com/mediapipe.SplitVectorCalculatorOptions] {
+      ranges: { begin: 16 end: 17 }
+    }
+  }
+}
+
+# Gets pose landmarks after wrists.
+node {
+  calculator: "SwitchContainer"
+  input_side_packet: "ENABLE:upper_body_only"
+  input_stream: "landmarks"
+  output_stream: "landmarks_after_wrist"
+  node_options: {
+    [type.googleapis.com/mediapipe.SwitchContainerOptions] {
+      contained_node: {
+        calculator: "SplitNormalizedLandmarkListCalculator"
+        node_options: {
+          [type.googleapis.com/mediapipe.SplitVectorCalculatorOptions] {
+            ranges: { begin: 23 end: 33 }
+          }
+        }
+      }
+      contained_node: {
+        calculator: "SplitNormalizedLandmarkListCalculator"
+        node_options: {
+          [type.googleapis.com/mediapipe.SplitVectorCalculatorOptions] {
+            ranges: { begin: 23 end: 25 }
+          }
+        }
+      }
+    }
+  }
+}
+
+# Gets left hand wrist landmark.
+node {
+  calculator: "HandWristForPose"
+  input_stream: "HAND_LANDMARKS:left_hand_landmarks"
+  output_stream: "WRIST_LANDMARK:left_hand_wrist_landmark"
+}
+
+# Gets left hand wrist landmark or keep pose wrist landmark if hand was not
+# predicted.
+node {
+  calculator: "MergeCalculator"
+  input_stream: "left_hand_wrist_landmark"
+  input_stream: "landmarks_left_wrist"
+  output_stream: "merged_left_hand_wrist_landmark"
+}
+
+# Gets right hand wrist landmark.
+node {
+  calculator: "HandWristForPose"
+  input_stream: "HAND_LANDMARKS:right_hand_landmarks"
+  output_stream: "WRIST_LANDMARK:right_hand_wrist_landmark"
+}
+
+# Gets right hand wrist landmark or keep pose wrist landmark if hand was not
+# predicted.
+node {
+  calculator: "MergeCalculator"
+  input_stream: "right_hand_wrist_landmark"
+  input_stream: "landmarks_right_wrist"
+  output_stream: "merged_right_hand_wrist_landmark"
+}
+
+# Combines pose landmarks all together.
+node {
+  calculator: "ConcatenateNormalizedLandmarkListCalculator"
+  input_stream: "landmarks_before_wrist"
+  input_stream: "merged_left_hand_wrist_landmark"
+  input_stream: "merged_right_hand_wrist_landmark"
+  input_stream: "landmarks_after_wrist"
+  output_stream: "landmarks_merged"
+  node_options: {
+    [type.googleapis.com/mediapipe.ConcatenateVectorCalculatorOptions] {
+      only_emit_if_all_present: true
+    }
+  }
+}
+
+# Takes left pose landmarks.
+node {
+  calculator: "SwitchContainer"
+  input_side_packet: "ENABLE:upper_body_only"
+  input_stream: "landmarks_merged"
+  output_stream: "landmarks_left_side"
+  node_options: {
+    [type.googleapis.com/mediapipe.SwitchContainerOptions] {
+      contained_node: {
+        calculator: "SplitNormalizedLandmarkListCalculator"
+        node_options: {
+          [type.googleapis.com/mediapipe.SplitVectorCalculatorOptions] {
+            ranges: { begin: 0 end: 1 }
+            ranges: { begin: 2 end: 3 }
+            ranges: { begin: 4 end: 5 }
+            ranges: { begin: 6 end: 7 }
+            ranges: { begin: 8 end: 9 }
+            ranges: { begin: 10 end: 11 }
+            ranges: { begin: 12 end: 13 }
+            ranges: { begin: 14 end: 15 }
+            combine_outputs: true
+          }
+        }
+      }
+      contained_node: {
+        calculator: "SplitNormalizedLandmarkListCalculator"
+        node_options: {
+          [type.googleapis.com/mediapipe.SplitVectorCalculatorOptions] {
+            ranges: { begin: 0 end: 1 }
+            ranges: { begin: 2 end: 3 }
+            ranges: { begin: 4 end: 5 }
+            ranges: { begin: 6 end: 7 }
+            combine_outputs: true
+          }
+        }
+      }
+    }
+  }
+}
+
+# Takes right pose landmarks.
+node {
+  calculator: "SwitchContainer"
+  input_side_packet: "ENABLE:upper_body_only"
+  input_stream: "landmarks_merged"
+  output_stream: "landmarks_right_side"
+  node_options: {
+    [type.googleapis.com/mediapipe.SwitchContainerOptions] {
+      contained_node: {
+        calculator: "SplitNormalizedLandmarkListCalculator"
+        node_options: {
+          [type.googleapis.com/mediapipe.SplitVectorCalculatorOptions] {
+            ranges: { begin: 1 end: 2 }
+            ranges: { begin: 3 end: 4 }
+            ranges: { begin: 5 end: 6 }
+            ranges: { begin: 7 end: 8 }
+            ranges: { begin: 9 end: 10 }
+            ranges: { begin: 11 end: 12 }
+            ranges: { begin: 13 end: 14 }
+            ranges: { begin: 15 end: 16 }
+            combine_outputs: true
+          }
+        }
+      }
+      contained_node: {
+        calculator: "SplitNormalizedLandmarkListCalculator"
+        node_options: {
+          [type.googleapis.com/mediapipe.SplitVectorCalculatorOptions] {
+            ranges: { begin: 1 end: 2 }
+            ranges: { begin: 3 end: 4 }
+            ranges: { begin: 5 end: 6 }
+            ranges: { begin: 7 end: 8 }
+            combine_outputs: true
+          }
+        }
+      }
+    }
+  }
+}
+
+# --------------------------------------------------------------------------- #
+# ---------------------------------- Pose ----------------------------------- #
+# --------------------------------------------------------------------------- #
+
+# Converts pose connections to white lines.
+node {
+  calculator: "SwitchContainer"
+  input_side_packet: "ENABLE:upper_body_only"
+  input_stream: "NORM_LANDMARKS:landmarks_merged"
+  input_stream: "RENDER_SCALE:render_scale"
+  output_stream: "RENDER_DATA:landmarks_render_data"
+  node_options: {
+    [type.googleapis.com/mediapipe.SwitchContainerOptions] {
+      contained_node: {
+        calculator: "LandmarksToRenderDataCalculator"
+        node_options: {
+          [type.googleapis.com/mediapipe.LandmarksToRenderDataCalculatorOptions] {
+            landmark_connections: 0
+            landmark_connections: 1
+            landmark_connections: 0
+            landmark_connections: 2
+            landmark_connections: 2
+            landmark_connections: 4
+            landmark_connections: 1
+            landmark_connections: 3
+            landmark_connections: 3
+            landmark_connections: 5
+            landmark_connections: 0
+            landmark_connections: 6
+            landmark_connections: 1
+            landmark_connections: 7
+            landmark_connections: 6
+            landmark_connections: 7
+            landmark_connections: 6
+            landmark_connections: 8
+            landmark_connections: 7
+            landmark_connections: 9
+            landmark_connections: 8
+            landmark_connections: 10
+            landmark_connections: 9
+            landmark_connections: 11
+            landmark_connections: 10
+            landmark_connections: 12
+            landmark_connections: 11
+            landmark_connections: 13
+            landmark_connections: 12
+            landmark_connections: 14
+            landmark_connections: 13
+            landmark_connections: 15
+            landmark_connections: 10
+            landmark_connections: 14
+            landmark_connections: 11
+            landmark_connections: 15
+
+            landmark_color { r: 255 g: 255 b: 255 }
+            connection_color { r: 255 g: 255 b: 255 }
+            thickness: 3.0
+            visualize_landmark_depth: false
+            utilize_visibility: true
+            visibility_threshold: 0.1
+          }
+        }
+      }
+      contained_node: {
+        calculator: "LandmarksToRenderDataCalculator"
+        node_options: {
+          [type.googleapis.com/mediapipe.LandmarksToRenderDataCalculatorOptions] {
+            landmark_connections: 0
+            landmark_connections: 1
+            landmark_connections: 0
+            landmark_connections: 2
+            landmark_connections: 2
+            landmark_connections: 4
+            landmark_connections: 1
+            landmark_connections: 3
+            landmark_connections: 3
+            landmark_connections: 5
+            landmark_connections: 0
+            landmark_connections: 6
+            landmark_connections: 1
+            landmark_connections: 7
+            landmark_connections: 6
+            landmark_connections: 7
+            landmark_color { r: 255 g: 255 b: 255 }
+            connection_color { r: 255 g: 255 b: 255 }
+            thickness: 3.0
+            visualize_landmark_depth: false
+            utilize_visibility: true
+            visibility_threshold: 0.5
+          }
+        }
+      }
+    }
+  }
+}
+
+# Converts pose joints to big white circles.
+node {
+  calculator: "LandmarksToRenderDataCalculator"
+  input_stream: "NORM_LANDMARKS:landmarks_merged"
+  input_stream: "RENDER_SCALE:render_scale"
+  output_stream: "RENDER_DATA:landmarks_background_joints_render_data"
+  node_options: {
+    [type.googleapis.com/mediapipe.LandmarksToRenderDataCalculatorOptions] {
+      landmark_color { r: 255 g: 255 b: 255 }
+      connection_color { r: 255 g: 255 b: 255 }
+      thickness: 5.0
+      visualize_landmark_depth: false
+      utilize_visibility: true
+      visibility_threshold: 0.5
+    }
+  }
+}
+
+# Converts pose left side joints to orange circles (inside white ones).
+node {
+  calculator: "LandmarksToRenderDataCalculator"
+  input_stream: "NORM_LANDMARKS:landmarks_left_side"
+  input_stream: "RENDER_SCALE:render_scale"
+  output_stream: "RENDER_DATA:landmarks_left_joints_render_data"
+  node_options: {
+    [type.googleapis.com/mediapipe.LandmarksToRenderDataCalculatorOptions] {
+      landmark_color { r: 255 g: 138 b: 0 }
+      connection_color { r: 255 g: 138 b: 0 }
+      thickness: 3.0
+      visualize_landmark_depth: false
+      utilize_visibility: true
+      visibility_threshold: 0.5
+    }
+  }
+}
+
+# Converts pose right side joints to cyan circles (inside white ones).
+node {
+  calculator: "LandmarksToRenderDataCalculator"
+  input_stream: "NORM_LANDMARKS:landmarks_right_side"
+  input_stream: "RENDER_SCALE:render_scale"
+  output_stream: "RENDER_DATA:landmarks_right_joints_render_data"
+  node_options: {
+    [type.googleapis.com/mediapipe.LandmarksToRenderDataCalculatorOptions] {
+      landmark_color { r: 0 g: 217 b: 231 }
+      connection_color { r: 0 g: 217 b: 231 }
+      thickness: 3.0
+      visualize_landmark_depth: false
+      utilize_visibility: true
+      visibility_threshold: 0.5
+    }
+  }
+}
+
+# --------------------------------------------------------------------------- #
+# ------------------------------- Left hand --------------------------------- #
+# --------------------------------------------------------------------------- #
+
+# Converts left hand connections to white lines.
+node {
+  calculator: "LandmarksToRenderDataCalculator"
+  input_stream: "NORM_LANDMARKS:left_hand_landmarks"
+  input_stream: "RENDER_SCALE:render_scale"
+  output_stream: "RENDER_DATA:left_hand_landmarks_connections_rd"
+  node_options: {
+    [type.googleapis.com/mediapipe.LandmarksToRenderDataCalculatorOptions] {
+      landmark_connections: 0
+      landmark_connections: 1
+      landmark_connections: 1
+      landmark_connections: 2
+      landmark_connections: 2
+      landmark_connections: 3
+      landmark_connections: 3
+      landmark_connections: 4
+      landmark_connections: 0
+      landmark_connections: 5
+      landmark_connections: 5
+      landmark_connections: 6
+      landmark_connections: 6
+      landmark_connections: 7
+      landmark_connections: 7
+      landmark_connections: 8
+      landmark_connections: 5
+      landmark_connections: 9
+      landmark_connections: 9
+      landmark_connections: 10
+      landmark_connections: 10
+      landmark_connections: 11
+      landmark_connections: 11
+      landmark_connections: 12
+      landmark_connections: 9
+      landmark_connections: 13
+      landmark_connections: 13
+      landmark_connections: 14
+      landmark_connections: 14
+      landmark_connections: 15
+      landmark_connections: 15
+      landmark_connections: 16
+      landmark_connections: 13
+      landmark_connections: 17
+      landmark_connections: 0
+      landmark_connections: 17
+      landmark_connections: 17
+      landmark_connections: 18
+      landmark_connections: 18
+      landmark_connections: 19
+      landmark_connections: 19
+      landmark_connections: 20
+      landmark_color { r: 255 g: 255 b: 255 }
+      connection_color { r: 255 g: 255 b: 255 }
+      thickness: 4.0
+      visualize_landmark_depth: false
+    }
+  }
+}
+
+# Converts left hand color joints.
+node {
+  calculator: "LandmarksToRenderDataCalculator"
+  input_stream: "NORM_LANDMARKS:left_hand_landmarks"
+  input_stream: "RENDER_SCALE:render_scale"
+  output_stream: "RENDER_DATA:left_hand_landmarks_joints_rd"
+  node_options: {
+    [type.googleapis.com/mediapipe.LandmarksToRenderDataCalculatorOptions] {
+      landmark_color { r: 255 g: 138 b: 0 }
+      connection_color { r: 255 g: 138 b: 0 }
+      thickness: 3.0
+      visualize_landmark_depth: false
+    }
+  }
+}
+
+# --------------------------------------------------------------------------- #
+# -------------------------------- Right hand ------------------------------- #
+# --------------------------------------------------------------------------- #
+
+# Converts right hand connections to white lines.
+node {
+  calculator: "LandmarksToRenderDataCalculator"
+  input_stream: "NORM_LANDMARKS:right_hand_landmarks"
+  input_stream: "RENDER_SCALE:render_scale"
+  output_stream: "RENDER_DATA:right_hand_landmarks_connections_rd"
+  node_options: {
+    [type.googleapis.com/mediapipe.LandmarksToRenderDataCalculatorOptions] {
+      landmark_connections: 0
+      landmark_connections: 1
+      landmark_connections: 1
+      landmark_connections: 2
+      landmark_connections: 2
+      landmark_connections: 3
+      landmark_connections: 3
+      landmark_connections: 4
+      landmark_connections: 0
+      landmark_connections: 5
+      landmark_connections: 5
+      landmark_connections: 6
+      landmark_connections: 6
+      landmark_connections: 7
+      landmark_connections: 7
+      landmark_connections: 8
+      landmark_connections: 5
+      landmark_connections: 9
+      landmark_connections: 9
+      landmark_connections: 10
+      landmark_connections: 10
+      landmark_connections: 11
+      landmark_connections: 11
+      landmark_connections: 12
+      landmark_connections: 9
+      landmark_connections: 13
+      landmark_connections: 13
+      landmark_connections: 14
+      landmark_connections: 14
+      landmark_connections: 15
+      landmark_connections: 15
+      landmark_connections: 16
+      landmark_connections: 13
+      landmark_connections: 17
+      landmark_connections: 0
+      landmark_connections: 17
+      landmark_connections: 17
+      landmark_connections: 18
+      landmark_connections: 18
+      landmark_connections: 19
+      landmark_connections: 19
+      landmark_connections: 20
+      landmark_color { r: 255 g: 255 b: 255 }
+      connection_color { r: 255 g: 255 b: 255 }
+      thickness: 4.0
+      visualize_landmark_depth: false
+    }
+  }
+}
+
+# Converts right hand color joints.
+node {
+  calculator: "LandmarksToRenderDataCalculator"
+  input_stream: "NORM_LANDMARKS:right_hand_landmarks"
+  input_stream: "RENDER_SCALE:render_scale"
+  output_stream: "RENDER_DATA:right_hand_landmarks_joints_rd"
+  node_options: {
+    [type.googleapis.com/mediapipe.LandmarksToRenderDataCalculatorOptions] {
+      landmark_color { r: 0 g: 217 b: 231 }
+      connection_color { r: 0 g: 217 b: 231 }
+      thickness: 3.0
+      visualize_landmark_depth: false
+    }
+  }
+}
+
+# --------------------------------------------------------------------------- #
+# ---------------------------------- Face ----------------------------------- #
+# --------------------------------------------------------------------------- #
+
+# Converts face connections to white lines.
+node {
+  calculator: "LandmarksToRenderDataCalculator"
+  input_stream: "NORM_LANDMARKS:face_landmarks"
+  input_stream: "RENDER_SCALE:render_scale"
+  output_stream: "RENDER_DATA:face_landmarks_connections_rd"
+  node_options: {
+    [type.googleapis.com/mediapipe.LandmarksToRenderDataCalculatorOptions] {
+      # Lips.
+      landmark_connections: 61
+      landmark_connections: 146
+      landmark_connections: 146
+      landmark_connections: 91
+      landmark_connections: 91
+      landmark_connections: 181
+      landmark_connections: 181
+      landmark_connections: 84
+      landmark_connections: 84
+      landmark_connections: 17
+      landmark_connections: 17
+      landmark_connections: 314
+      landmark_connections: 314
+      landmark_connections: 405
+      landmark_connections: 405
+      landmark_connections: 321
+      landmark_connections: 321
+      landmark_connections: 375
+      landmark_connections: 375
+      landmark_connections: 291
+      landmark_connections: 61
+      landmark_connections: 185
+      landmark_connections: 185
+      landmark_connections: 40
+      landmark_connections: 40
+      landmark_connections: 39
+      landmark_connections: 39
+      landmark_connections: 37
+      landmark_connections: 37
+      landmark_connections: 0
+      landmark_connections: 0
+      landmark_connections: 267
+      landmark_connections: 267
+      landmark_connections: 269
+      landmark_connections: 269
+      landmark_connections: 270
+      landmark_connections: 270
+      landmark_connections: 409
+      landmark_connections: 409
+      landmark_connections: 291
+      landmark_connections: 78
+      landmark_connections: 95
+      landmark_connections: 95
+      landmark_connections: 88
+      landmark_connections: 88
+      landmark_connections: 178
+      landmark_connections: 178
+      landmark_connections: 87
+      landmark_connections: 87
+      landmark_connections: 14
+      landmark_connections: 14
+      landmark_connections: 317
+      landmark_connections: 317
+      landmark_connections: 402
+      landmark_connections: 402
+      landmark_connections: 318
+      landmark_connections: 318
+      landmark_connections: 324
+      landmark_connections: 324
+      landmark_connections: 308
+      landmark_connections: 78
+      landmark_connections: 191
+      landmark_connections: 191
+      landmark_connections: 80
+      landmark_connections: 80
+      landmark_connections: 81
+      landmark_connections: 81
+      landmark_connections: 82
+      landmark_connections: 82
+      landmark_connections: 13
+      landmark_connections: 13
+      landmark_connections: 312
+      landmark_connections: 312
+      landmark_connections: 311
+      landmark_connections: 311
+      landmark_connections: 310
+      landmark_connections: 310
+      landmark_connections: 415
+      landmark_connections: 415
+      landmark_connections: 308
+      # Left eye.
+      landmark_connections: 33
+      landmark_connections: 7
+      landmark_connections: 7
+      landmark_connections: 163
+      landmark_connections: 163
+      landmark_connections: 144
+      landmark_connections: 144
+      landmark_connections: 145
+      landmark_connections: 145
+      landmark_connections: 153
+      landmark_connections: 153
+      landmark_connections: 154
+      landmark_connections: 154
+      landmark_connections: 155
+      landmark_connections: 155
+      landmark_connections: 133
+      landmark_connections: 33
+      landmark_connections: 246
+      landmark_connections: 246
+      landmark_connections: 161
+      landmark_connections: 161
+      landmark_connections: 160
+      landmark_connections: 160
+      landmark_connections: 159
+      landmark_connections: 159
+      landmark_connections: 158
+      landmark_connections: 158
+      landmark_connections: 157
+      landmark_connections: 157
+      landmark_connections: 173
+      landmark_connections: 173
+      landmark_connections: 133
+      # Left eyebrow.
+      landmark_connections: 46
+      landmark_connections: 53
+      landmark_connections: 53
+      landmark_connections: 52
+      landmark_connections: 52
+      landmark_connections: 65
+      landmark_connections: 65
+      landmark_connections: 55
+      landmark_connections: 70
+      landmark_connections: 63
+      landmark_connections: 63
+      landmark_connections: 105
+      landmark_connections: 105
+      landmark_connections: 66
+      landmark_connections: 66
+      landmark_connections: 107
+      # Right eye.
+      landmark_connections: 263
+      landmark_connections: 249
+      landmark_connections: 249
+      landmark_connections: 390
+      landmark_connections: 390
+      landmark_connections: 373
+      landmark_connections: 373
+      landmark_connections: 374
+      landmark_connections: 374
+      landmark_connections: 380
+      landmark_connections: 380
+      landmark_connections: 381
+      landmark_connections: 381
+      landmark_connections: 382
+      landmark_connections: 382
+      landmark_connections: 362
+      landmark_connections: 263
+      landmark_connections: 466
+      landmark_connections: 466
+      landmark_connections: 388
+      landmark_connections: 388
+      landmark_connections: 387
+      landmark_connections: 387
+      landmark_connections: 386
+      landmark_connections: 386
+      landmark_connections: 385
+      landmark_connections: 385
+      landmark_connections: 384
+      landmark_connections: 384
+      landmark_connections: 398
+      landmark_connections: 398
+      landmark_connections: 362
+      # Right eyebrow.
+      landmark_connections: 276
+      landmark_connections: 283
+      landmark_connections: 283
+      landmark_connections: 282
+      landmark_connections: 282
+      landmark_connections: 295
+      landmark_connections: 295
+      landmark_connections: 285
+      landmark_connections: 300
+      landmark_connections: 293
+      landmark_connections: 293
+      landmark_connections: 334
+      landmark_connections: 334
+      landmark_connections: 296
+      landmark_connections: 296
+      landmark_connections: 336
+      # Face oval.
+      landmark_connections: 10
+      landmark_connections: 338
+      landmark_connections: 338
+      landmark_connections: 297
+      landmark_connections: 297
+      landmark_connections: 332
+      landmark_connections: 332
+      landmark_connections: 284
+      landmark_connections: 284
+      landmark_connections: 251
+      landmark_connections: 251
+      landmark_connections: 389
+      landmark_connections: 389
+      landmark_connections: 356
+      landmark_connections: 356
+      landmark_connections: 454
+      landmark_connections: 454
+      landmark_connections: 323
+      landmark_connections: 323
+      landmark_connections: 361
+      landmark_connections: 361
+      landmark_connections: 288
+      landmark_connections: 288
+      landmark_connections: 397
+      landmark_connections: 397
+      landmark_connections: 365
+      landmark_connections: 365
+      landmark_connections: 379
+      landmark_connections: 379
+      landmark_connections: 378
+      landmark_connections: 378
+      landmark_connections: 400
+      landmark_connections: 400
+      landmark_connections: 377
+      landmark_connections: 377
+      landmark_connections: 152
+      landmark_connections: 152
+      landmark_connections: 148
+      landmark_connections: 148
+      landmark_connections: 176
+      landmark_connections: 176
+      landmark_connections: 149
+      landmark_connections: 149
+      landmark_connections: 150
+      landmark_connections: 150
+      landmark_connections: 136
+      landmark_connections: 136
+      landmark_connections: 172
+      landmark_connections: 172
+      landmark_connections: 58
+      landmark_connections: 58
+      landmark_connections: 132
+      landmark_connections: 132
+      landmark_connections: 93
+      landmark_connections: 93
+      landmark_connections: 234
+      landmark_connections: 234
+      landmark_connections: 127
+      landmark_connections: 127
+      landmark_connections: 162
+      landmark_connections: 162
+      landmark_connections: 21
+      landmark_connections: 21
+      landmark_connections: 54
+      landmark_connections: 54
+      landmark_connections: 103
+      landmark_connections: 103
+      landmark_connections: 67
+      landmark_connections: 67
+      landmark_connections: 109
+      landmark_connections: 109
+      landmark_connections: 10
+      landmark_color { r: 255 g: 255 b: 255 }
+      connection_color { r: 255 g: 255 b: 255 }
+      thickness: 0.5
+      visualize_landmark_depth: false
+    }
+  }
+}
+
+# Converts face joints to cyan circles.
+node {
+  calculator: "LandmarksToRenderDataCalculator"
+  input_stream: "NORM_LANDMARKS:face_landmarks"
+  input_stream: "RENDER_SCALE:render_scale"
+  output_stream: "RENDER_DATA:face_landmarks_joints_rd"
+  node_options: {
+    [type.googleapis.com/mediapipe.LandmarksToRenderDataCalculatorOptions] {
+      landmark_color { r: 0 g: 217 b: 231 }
+      connection_color { r: 0 g: 217 b: 231 }
+      thickness: 0.5
+      visualize_landmark_depth: false
+    }
+  }
+}
+
+# Concatenates all render data.
+node {
+  calculator: "ConcatenateRenderDataVectorCalculator"
+  input_stream: "landmarks_render_data"
+  input_stream: "landmarks_background_joints_render_data"
+  input_stream: "landmarks_left_joints_render_data"
+  input_stream: "landmarks_right_joints_render_data"
+
+  # Left hand.
+  input_stream: "left_hand_landmarks_connections_rd"
+  input_stream: "left_hand_landmarks_joints_rd"
+
+  # Right hand.
+  input_stream: "right_hand_landmarks_connections_rd"
+  input_stream: "right_hand_landmarks_joints_rd"
+
+  # Face.
+  input_stream: "face_landmarks_connections_rd"
+  input_stream: "face_landmarks_joints_rd"
+
+  output_stream: "render_data_vector"
+}

mediapipe/graphs/instant_motion_tracking/calculators/BUILD

@@ -61,8 +61,8 @@ cc_library(
         "//mediapipe/framework/port:opencv_imgproc",
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework/port:status",
-        "//mediapipe/graphs/object_detection_3d/calculators:box",
         "//mediapipe/graphs/object_detection_3d/calculators:model_matrix_cc_proto",
+        "//mediapipe/modules/objectron/calculators:box",
         "@com_google_absl//absl/memory",
         "@com_google_absl//absl/strings",
         "@eigen_archive//:eigen",

mediapipe/graphs/instant_motion_tracking/calculators/matrices_manager_calculator.cc

@@ -25,8 +25,8 @@
 #include "mediapipe/framework/port/ret_check.h"
 #include "mediapipe/framework/port/status.h"
 #include "mediapipe/graphs/instant_motion_tracking/calculators/transformations.h"
-#include "mediapipe/graphs/object_detection_3d/calculators/box.h"
 #include "mediapipe/graphs/object_detection_3d/calculators/model_matrix.pb.h"
+#include "mediapipe/modules/objectron/calculators/box.h"
 
 namespace mediapipe {
 
@@ -87,9 +87,9 @@ constexpr float kInitialZ = -10.0f;
 
 class MatricesManagerCalculator : public CalculatorBase {
  public:
-  static ::mediapipe::Status GetContract(CalculatorContract* cc);
-  ::mediapipe::Status Open(CalculatorContext* cc) override;
-  ::mediapipe::Status Process(CalculatorContext* cc) override;
+  static mediapipe::Status GetContract(CalculatorContract* cc);
+  mediapipe::Status Open(CalculatorContext* cc) override;
+  mediapipe::Status Process(CalculatorContext* cc) override;
 
  private:
   // Device properties that will be preset by side packets
@@ -137,7 +137,7 @@ class MatricesManagerCalculator : public CalculatorBase {
 
 REGISTER_CALCULATOR(MatricesManagerCalculator);
 
-::mediapipe::Status MatricesManagerCalculator::GetContract(
+mediapipe::Status MatricesManagerCalculator::GetContract(
     CalculatorContract* cc) {
   RET_CHECK(cc->Inputs().HasTag(kAnchorsTag) &&
             cc->Inputs().HasTag(kIMUMatrixTag) &&
@@ -162,20 +162,20 @@ REGISTER_CALCULATOR(MatricesManagerCalculator);
   cc->InputSidePackets().Tag(kFOVSidePacketTag).Set<float>();
   cc->InputSidePackets().Tag(kAspectRatioSidePacketTag).Set<float>();
 
-  return ::mediapipe::OkStatus();
+  return mediapipe::OkStatus();
 }
 
-::mediapipe::Status MatricesManagerCalculator::Open(CalculatorContext* cc) {
+mediapipe::Status MatricesManagerCalculator::Open(CalculatorContext* cc) {
   cc->SetOffset(TimestampDiff(0));
   // Set device properties from side packets
   vertical_fov_radians_ =
       cc->InputSidePackets().Tag(kFOVSidePacketTag).Get<float>();
   aspect_ratio_ =
       cc->InputSidePackets().Tag(kAspectRatioSidePacketTag).Get<float>();
-  return ::mediapipe::OkStatus();
+  return mediapipe::OkStatus();
 }
 
-::mediapipe::Status MatricesManagerCalculator::Process(CalculatorContext* cc) {
+mediapipe::Status MatricesManagerCalculator::Process(CalculatorContext* cc) {
   // Define each object's model matrices
   auto asset_matrices_gif =
       std::make_unique<mediapipe::TimedModelMatrixProtoList>();
@@ -276,7 +276,7 @@ REGISTER_CALCULATOR(MatricesManagerCalculator);
       .Get(cc->Outputs().GetId("MATRICES", 1))
       .Add(asset_matrices_1.release(), cc->InputTimestamp());
 
-  return ::mediapipe::OkStatus();
+  return mediapipe::OkStatus();
 }
 
 // Using a specified rotation value in radians, generate a rotation matrix for

mediapipe/graphs/instant_motion_tracking/calculators/sticker_manager_calculator.cc

@@ -53,7 +53,7 @@ constexpr char kRenderDescriptorsTag[] = "RENDER_DATA";
 
 class StickerManagerCalculator : public CalculatorBase {
  public:
-  static ::mediapipe::Status GetContract(CalculatorContract* cc) {
+  static mediapipe::Status GetContract(CalculatorContract* cc) {
     RET_CHECK(cc->Inputs().HasTag(kProtoDataString));
     RET_CHECK(cc->Outputs().HasTag(kAnchorsTag) &&
               cc->Outputs().HasTag(kUserRotationsTag) &&
@@ -66,15 +66,15 @@ class StickerManagerCalculator : public CalculatorBase {
     cc->Outputs().Tag(kUserScalingsTag).Set<std::vector<UserScaling>>();
     cc->Outputs().Tag(kRenderDescriptorsTag).Set<std::vector<int>>();
 
-    return ::mediapipe::OkStatus();
+    return mediapipe::OkStatus();
   }
 
-  ::mediapipe::Status Open(CalculatorContext* cc) override {
+  mediapipe::Status Open(CalculatorContext* cc) override {
     cc->SetOffset(TimestampDiff(0));
-    return ::mediapipe::OkStatus();
+    return mediapipe::OkStatus();
   }
 
-  ::mediapipe::Status Process(CalculatorContext* cc) override {
+  mediapipe::Status Process(CalculatorContext* cc) override {
     std::string sticker_proto_string =
         cc->Inputs().Tag(kProtoDataString).Get<std::string>();
 
@@ -138,11 +138,11 @@ class StickerManagerCalculator : public CalculatorBase {
                          .At(cc->InputTimestamp()));
     }
 
-    return ::mediapipe::OkStatus();
+    return mediapipe::OkStatus();
   }
 
-  ::mediapipe::Status Close(CalculatorContext* cc) override {
-    return ::mediapipe::OkStatus();
+  mediapipe::Status Close(CalculatorContext* cc) override {
+    return mediapipe::OkStatus();
   }
 };
 

mediapipe/graphs/instant_motion_tracking/calculators/tracked_anchor_manager_calculator.cc

@@ -71,7 +71,7 @@ class TrackedAnchorManagerCalculator : public CalculatorBase {
   std::vector<Anchor> previous_anchor_data_;
 
  public:
-  static ::mediapipe::Status GetContract(CalculatorContract* cc) {
+  static mediapipe::Status GetContract(CalculatorContract* cc) {
     RET_CHECK(cc->Inputs().HasTag(kAnchorsTag) &&
               cc->Inputs().HasTag(kSentinelTag));
     RET_CHECK(cc->Outputs().HasTag(kAnchorsTag) &&
@@ -91,18 +91,18 @@ class TrackedAnchorManagerCalculator : public CalculatorBase {
       cc->Outputs().Tag(kCancelTag).Set<int>();
     }
 
-    return ::mediapipe::OkStatus();
+    return mediapipe::OkStatus();
   }
 
-  ::mediapipe::Status Open(CalculatorContext* cc) override {
-    return ::mediapipe::OkStatus();
+  mediapipe::Status Open(CalculatorContext* cc) override {
+    return mediapipe::OkStatus();
   }
 
-  ::mediapipe::Status Process(CalculatorContext* cc) override;
+  mediapipe::Status Process(CalculatorContext* cc) override;
 };
 REGISTER_CALCULATOR(TrackedAnchorManagerCalculator);
 
-::mediapipe::Status TrackedAnchorManagerCalculator::Process(
+mediapipe::Status TrackedAnchorManagerCalculator::Process(
     CalculatorContext* cc) {
   mediapipe::Timestamp timestamp = cc->InputTimestamp();
   const int sticker_sentinel = cc->Inputs().Tag(kSentinelTag).Get<int>();
@@ -208,6 +208,6 @@ REGISTER_CALCULATOR(TrackedAnchorManagerCalculator);
       .Tag(kBoxesOutputTag)
       .Add(pos_boxes.release(), cc->InputTimestamp());
 
-  return ::mediapipe::OkStatus();
+  return mediapipe::OkStatus();
 }
 }  // namespace mediapipe

mediapipe/graphs/iris_tracking/calculators/iris_to_depth_calculator.cc

@@ -89,7 +89,7 @@ float CalculateDepth(const NormalizedLandmark& center, float focal_length,
 // }
 class IrisToDepthCalculator : public CalculatorBase {
  public:
-  static ::mediapipe::Status GetContract(CalculatorContract* cc) {
+  static mediapipe::Status GetContract(CalculatorContract* cc) {
     cc->Inputs().Tag(kIrisTag).Set<NormalizedLandmarkList>();
     cc->Inputs().Tag(kImageSizeTag).Set<std::pair<int, int>>();
 
@@ -111,12 +111,12 @@ class IrisToDepthCalculator : public CalculatorBase {
     if (cc->Outputs().HasTag(kRightIrisDepthTag)) {
       cc->Outputs().Tag(kRightIrisDepthTag).Set<float>();
     }
-    return ::mediapipe::OkStatus();
+    return mediapipe::OkStatus();
   }
 
-  ::mediapipe::Status Open(CalculatorContext* cc) override;
+  mediapipe::Status Open(CalculatorContext* cc) override;
 
-  ::mediapipe::Status Process(CalculatorContext* cc) override;
+  mediapipe::Status Process(CalculatorContext* cc) override;
 
  private:
   float focal_length_pixels_ = -1.f;
@@ -134,7 +134,7 @@ class IrisToDepthCalculator : public CalculatorBase {
 };
 REGISTER_CALCULATOR(IrisToDepthCalculator);
 
-::mediapipe::Status IrisToDepthCalculator::Open(CalculatorContext* cc) {
+mediapipe::Status IrisToDepthCalculator::Open(CalculatorContext* cc) {
   cc->SetOffset(TimestampDiff(0));
   if (cc->InputSidePackets().HasTag(kFocalLengthPixelTag)) {
 #if defined(__APPLE__)
@@ -155,13 +155,13 @@ REGISTER_CALCULATOR(IrisToDepthCalculator);
   }
 
   options_ = cc->Options<::mediapipe::IrisToDepthCalculatorOptions>();
-  return ::mediapipe::OkStatus();
+  return mediapipe::OkStatus();
 }
 
-::mediapipe::Status IrisToDepthCalculator::Process(CalculatorContext* cc) {
+mediapipe::Status IrisToDepthCalculator::Process(CalculatorContext* cc) {
   // Only process if there's input landmarks.
   if (cc->Inputs().Tag(kIrisTag).IsEmpty()) {
-    return ::mediapipe::OkStatus();
+    return mediapipe::OkStatus();
   }
 
   const auto& iris_landmarks =
@@ -220,7 +220,7 @@ REGISTER_CALCULATOR(IrisToDepthCalculator);
                          .At(cc->InputTimestamp()));
     }
   }
-  return ::mediapipe::OkStatus();
+  return mediapipe::OkStatus();
 }
 
 void IrisToDepthCalculator::GetLeftIris(const NormalizedLandmarkList& lds,

mediapipe/graphs/iris_tracking/calculators/iris_to_render_data_calculator.cc

@@ -108,7 +108,7 @@ float CalculateDepth(const NormalizedLandmark& center, float focal_length,
 // }
 class IrisToRenderDataCalculator : public CalculatorBase {
  public:
-  static ::mediapipe::Status GetContract(CalculatorContract* cc) {
+  static mediapipe::Status GetContract(CalculatorContract* cc) {
     cc->Inputs().Tag(kIrisTag).Set<NormalizedLandmarkList>();
     cc->Outputs().Tag(kRenderDataTag).Set<RenderData>();
     cc->Inputs().Tag(kImageSizeTag).Set<std::pair<int, int>>();
@@ -119,12 +119,12 @@ class IrisToRenderDataCalculator : public CalculatorBase {
     if (cc->Inputs().HasTag(kRightIrisDepthTag)) {
       cc->Inputs().Tag(kRightIrisDepthTag).Set<float>();
     }
-    return ::mediapipe::OkStatus();
+    return mediapipe::OkStatus();
   }
 
-  ::mediapipe::Status Open(CalculatorContext* cc) override;
+  mediapipe::Status Open(CalculatorContext* cc) override;
 
-  ::mediapipe::Status Process(CalculatorContext* cc) override;
+  mediapipe::Status Process(CalculatorContext* cc) override;
 
  private:
   void RenderIris(const NormalizedLandmarkList& iris_landmarks,
@@ -150,15 +150,15 @@ class IrisToRenderDataCalculator : public CalculatorBase {
 };
 REGISTER_CALCULATOR(IrisToRenderDataCalculator);
 
-::mediapipe::Status IrisToRenderDataCalculator::Open(CalculatorContext* cc) {
+mediapipe::Status IrisToRenderDataCalculator::Open(CalculatorContext* cc) {
   cc->SetOffset(TimestampDiff(0));
-  return ::mediapipe::OkStatus();
+  return mediapipe::OkStatus();
 }
 
-::mediapipe::Status IrisToRenderDataCalculator::Process(CalculatorContext* cc) {
+mediapipe::Status IrisToRenderDataCalculator::Process(CalculatorContext* cc) {
   // Only process if there's input landmarks.
   if (cc->Inputs().Tag(kIrisTag).IsEmpty()) {
-    return ::mediapipe::OkStatus();
+    return mediapipe::OkStatus();
   }
   const auto& options =
       cc->Options<::mediapipe::IrisToRenderDataCalculatorOptions>();
@@ -212,7 +212,7 @@ REGISTER_CALCULATOR(IrisToRenderDataCalculator);
   cc->Outputs()
       .Tag(kRenderDataTag)
       .Add(render_data.release(), cc->InputTimestamp());
-  return ::mediapipe::OkStatus();
+  return mediapipe::OkStatus();
 }
 
 void IrisToRenderDataCalculator::AddTextRenderData(

mediapipe/graphs/iris_tracking/calculators/update_face_landmarks_calculator.cc

@@ -215,28 +215,28 @@ constexpr int kEyeLandmarkIndicesInFaceLandmarks[] = {
 //
 class UpdateFaceLandmarksCalculator : public CalculatorBase {
  public:
-  static ::mediapipe::Status GetContract(CalculatorContract* cc) {
+  static mediapipe::Status GetContract(CalculatorContract* cc) {
     cc->Inputs().Tag(kFaceLandmarksTag).Set<NormalizedLandmarkList>();
     cc->Inputs().Tag(kNewEyeLandmarksTag).Set<NormalizedLandmarkList>();
 
     cc->Outputs().Tag(kUpdatedFaceLandmarksTag).Set<NormalizedLandmarkList>();
 
-    return ::mediapipe::OkStatus();
+    return mediapipe::OkStatus();
   }
-  ::mediapipe::Status Open(CalculatorContext* cc) {
+  mediapipe::Status Open(CalculatorContext* cc) {
     cc->SetOffset(TimestampDiff(0));
-    return ::mediapipe::OkStatus();
+    return mediapipe::OkStatus();
   }
 
-  ::mediapipe::Status Process(CalculatorContext* cc) override;
+  mediapipe::Status Process(CalculatorContext* cc) override;
 };
 REGISTER_CALCULATOR(UpdateFaceLandmarksCalculator);
 
-::mediapipe::Status UpdateFaceLandmarksCalculator::Process(
+mediapipe::Status UpdateFaceLandmarksCalculator::Process(
     CalculatorContext* cc) {
   if (cc->Inputs().Tag(kFaceLandmarksTag).IsEmpty() ||
       cc->Inputs().Tag(kNewEyeLandmarksTag).IsEmpty()) {
-    return ::mediapipe::OkStatus();
+    return mediapipe::OkStatus();
   }
   const auto& face_landmarks =
       cc->Inputs().Tag(kFaceLandmarksTag).Get<NormalizedLandmarkList>();
@@ -263,7 +263,7 @@ REGISTER_CALCULATOR(UpdateFaceLandmarksCalculator);
       .Tag(kUpdatedFaceLandmarksTag)
       .Add(refined_face_landmarks.release(), cc->InputTimestamp());
 
-  return ::mediapipe::OkStatus();
+  return mediapipe::OkStatus();
 }
 
 }  // namespace mediapipe

mediapipe/graphs/object_detection_3d/BUILD

@@ -29,16 +29,11 @@ cc_library(
     name = "mobile_calculators",
     visibility = ["//visibility:public"],
     deps = [
-        "//mediapipe/calculators/core:constant_side_packet_calculator",
         "//mediapipe/calculators/core:flow_limiter_calculator",
-        "//mediapipe/calculators/core:gate_calculator",
-        "//mediapipe/calculators/core:merge_calculator",
-        "//mediapipe/calculators/core:previous_loopback_calculator",
         "//mediapipe/calculators/image:image_cropping_calculator",
         "//mediapipe/graphs/object_detection_3d/calculators:annotations_to_model_matrices_calculator",
         "//mediapipe/graphs/object_detection_3d/calculators:gl_animation_overlay_calculator",
-        "//mediapipe/graphs/object_detection_3d/subgraphs:box_landmark_gpu",
-        "//mediapipe/graphs/object_detection_3d/subgraphs:object_detection_oid_v4_gpu",
+        "//mediapipe/modules/objectron:objectron_gpu",
     ],
 )
 
@@ -51,8 +46,21 @@ cc_library(
         "//mediapipe/gpu:gl_scaler_calculator",
         "//mediapipe/graphs/object_detection_3d/calculators:annotations_to_model_matrices_calculator",
         "//mediapipe/graphs/object_detection_3d/calculators:gl_animation_overlay_calculator",
-        "//mediapipe/graphs/object_detection_3d/subgraphs:objectron_detection_gpu",
-        "//mediapipe/graphs/object_detection_3d/subgraphs:objectron_tracking_gpu",
+        "//mediapipe/modules/objectron:objectron_detection_1stage_gpu",
+        "//mediapipe/modules/objectron:objectron_tracking_1stage_gpu",
+    ],
+)
+
+cc_library(
+    name = "desktop_cpu_calculators",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/calculators/tflite:tflite_model_calculator",
+        "//mediapipe/calculators/util:local_file_contents_calculator",
+        "//mediapipe/calculators/video:opencv_video_decoder_calculator",
+        "//mediapipe/calculators/video:opencv_video_encoder_calculator",
+        "//mediapipe/graphs/object_detection_3d/subgraphs:renderer_cpu",
+        "//mediapipe/modules/objectron:objectron_cpu",
     ],
 )
 

mediapipe/graphs/object_detection_3d/calculators/BUILD

@@ -18,43 +18,6 @@ licenses(["notice"])
 
 package(default_visibility = ["//visibility:public"])
 
-mediapipe_proto_library(
-    name = "object_proto",
-    srcs = ["object.proto"],
-    visibility = ["//visibility:public"],
-)
-
-mediapipe_proto_library(
-    name = "a_r_capture_metadata_proto",
-    srcs = ["a_r_capture_metadata.proto"],
-    visibility = ["//visibility:public"],
-)
-
-mediapipe_proto_library(
-    name = "annotation_proto",
-    srcs = ["annotation_data.proto"],
-    visibility = ["//visibility:public"],
-    deps = [
-        ":a_r_capture_metadata_proto",
-        ":object_proto",
-    ],
-)
-
-mediapipe_proto_library(
-    name = "camera_parameters_proto",
-    srcs = ["camera_parameters.proto"],
-    visibility = ["//visibility:public"],
-)
-
-mediapipe_proto_library(
-    name = "frame_annotation_tracker_calculator_proto",
-    srcs = ["frame_annotation_tracker_calculator.proto"],
-    visibility = ["//visibility:public"],
-    deps = [
-        "//mediapipe/framework:calculator_proto",
-    ],
-)
-
 mediapipe_proto_library(
     name = "gl_animation_overlay_calculator_proto",
     srcs = ["gl_animation_overlay_calculator.proto"],
@@ -64,32 +27,6 @@ mediapipe_proto_library(
     ],
 )
 
-mediapipe_proto_library(
-    name = "belief_decoder_config_proto",
-    srcs = ["belief_decoder_config.proto"],
-    visibility = ["//visibility:public"],
-)
-
-mediapipe_proto_library(
-    name = "tflite_tensors_to_objects_calculator_proto",
-    srcs = ["tflite_tensors_to_objects_calculator.proto"],
-    visibility = ["//visibility:public"],
-    deps = [
-        ":belief_decoder_config_proto",
-        "//mediapipe/framework:calculator_proto",
-    ],
-)
-
-mediapipe_proto_library(
-    name = "lift_2d_frame_annotation_to_3d_calculator_proto",
-    srcs = ["lift_2d_frame_annotation_to_3d_calculator.proto"],
-    visibility = ["//visibility:public"],
-    deps = [
-        ":belief_decoder_config_proto",
-        "//mediapipe/framework:calculator_proto",
-    ],
-)
-
 mediapipe_proto_library(
     name = "annotations_to_model_matrices_calculator_proto",
     srcs = ["annotations_to_model_matrices_calculator.proto"],
@@ -118,57 +55,11 @@ mediapipe_proto_library(
     ],
 )
 
-mediapipe_proto_library(
-    name = "frame_annotation_to_rect_calculator_proto",
-    srcs = ["frame_annotation_to_rect_calculator.proto"],
-    visibility = ["//visibility:public"],
-    deps = [
-        "//mediapipe/framework:calculator_proto",
-    ],
-)
-
-mediapipe_proto_library(
-    name = "filter_detection_calculator_proto",
-    srcs = ["filter_detection_calculator.proto"],
-    visibility = ["//visibility:public"],
-    deps = [
-        "//mediapipe/framework:calculator_proto",
-    ],
-)
-
-cc_library(
-    name = "box_util",
-    srcs = ["box_util.cc"],
-    hdrs = ["box_util.h"],
-    deps = [
-        "//mediapipe/framework/port:logging",
-        "//mediapipe/framework/port:opencv_core",
-        "//mediapipe/framework/port:opencv_imgproc",
-        "//mediapipe/util/tracking:box_tracker_cc_proto",
-    ],
-)
-
-cc_library(
-    name = "frame_annotation_tracker",
-    srcs = ["frame_annotation_tracker.cc"],
-    hdrs = ["frame_annotation_tracker.h"],
-    deps = [
-        ":annotation_cc_proto",
-        ":box_util",
-        "//mediapipe/framework/port:integral_types",
-        "//mediapipe/framework/port:logging",
-        "//mediapipe/util/tracking:box_tracker_cc_proto",
-        "@com_google_absl//absl/container:btree",
-        "@com_google_absl//absl/container:flat_hash_set",
-    ],
-)
-
 cc_library(
     name = "gl_animation_overlay_calculator",
     srcs = ["gl_animation_overlay_calculator.cc"],
     visibility = ["//visibility:public"],
     deps = [
-        ":camera_parameters_cc_proto",
         ":gl_animation_overlay_calculator_cc_proto",
         ":model_matrix_cc_proto",
         "//mediapipe/framework:calculator_framework",
@@ -176,164 +67,25 @@ cc_library(
         "//mediapipe/framework/port:status",
         "//mediapipe/gpu:gl_calculator_helper",
         "//mediapipe/gpu:shader_util",
+        "//mediapipe/modules/objectron/calculators:camera_parameters_cc_proto",
         "//mediapipe/util/android:asset_manager_util",
     ],
     alwayslink = 1,
 )
 
-cc_library(
-    name = "decoder",
-    srcs = [
-        "decoder.cc",
-    ],
-    hdrs = [
-        "decoder.h",
-    ],
-    deps = [
-        ":annotation_cc_proto",
-        ":belief_decoder_config_cc_proto",
-        "//mediapipe/framework/port:logging",
-        "//mediapipe/framework/port:opencv_core",
-        "//mediapipe/framework/port:opencv_imgproc",
-        "//mediapipe/framework/port:status",
-        "@com_google_absl//absl/status",
-        "@eigen_archive//:eigen",
-    ],
-)
-
-cc_library(
-    name = "tensor_util",
-    srcs = [
-        "tensor_util.cc",
-    ],
-    hdrs = [
-        "tensor_util.h",
-    ],
-    deps = [
-        "//mediapipe/framework/port:logging",
-        "//mediapipe/framework/port:opencv_core",
-        "@org_tensorflow//tensorflow/lite:framework",
-    ],
-)
-
-cc_library(
-    name = "box",
-    srcs = [
-        "box.cc",
-        "model.cc",
-    ],
-    hdrs = [
-        "box.h",
-        "model.h",
-        "types.h",
-    ],
-    visibility = ["//visibility:public"],
-    deps = [
-        ":annotation_cc_proto",
-        ":object_cc_proto",
-        "//mediapipe/framework/port:logging",
-        "@eigen_archive//:eigen",
-    ],
-)
-
-cc_library(
-    name = "frame_annotation_to_timed_box_list_calculator",
-    srcs = ["frame_annotation_to_timed_box_list_calculator.cc"],
-    visibility = ["//visibility:public"],
-    deps = [
-        ":annotation_cc_proto",
-        ":box_util",
-        "//mediapipe/framework:calculator_framework",
-        "//mediapipe/framework/port:opencv_core",
-        "//mediapipe/framework/port:opencv_imgproc",
-        "//mediapipe/framework/port:ret_check",
-        "//mediapipe/framework/port:status",
-        "//mediapipe/util/tracking:box_tracker_cc_proto",
-        "@com_google_absl//absl/memory",
-    ],
-    alwayslink = 1,
-)
-
-cc_library(
-    name = "frame_annotation_tracker_calculator",
-    srcs = ["frame_annotation_tracker_calculator.cc"],
-    visibility = ["//visibility:public"],
-    deps = [
-        ":annotation_cc_proto",
-        ":frame_annotation_tracker",
-        ":frame_annotation_tracker_calculator_cc_proto",
-        "//mediapipe/framework:calculator_framework",
-        "//mediapipe/framework/port:ret_check",
-        "//mediapipe/framework/port:status",
-        "//mediapipe/util/tracking:box_tracker_cc_proto",
-        "@com_google_absl//absl/container:flat_hash_set",
-        "@com_google_absl//absl/memory",
-    ],
-    alwayslink = 1,
-)
-
-cc_library(
-    name = "tflite_tensors_to_objects_calculator",
-    srcs = ["tflite_tensors_to_objects_calculator.cc"],
-    visibility = ["//visibility:public"],
-    deps = [
-        ":annotation_cc_proto",
-        ":belief_decoder_config_cc_proto",
-        ":decoder",
-        ":tensor_util",
-        ":tflite_tensors_to_objects_calculator_cc_proto",
-        "//mediapipe/framework:calculator_framework",
-        "//mediapipe/framework/deps:file_path",
-        "//mediapipe/framework/formats:detection_cc_proto",
-        "//mediapipe/framework/port:opencv_core",
-        "//mediapipe/framework/port:ret_check",
-        "@com_google_absl//absl/memory",
-        "@com_google_absl//absl/strings:str_format",
-        "@com_google_absl//absl/types:span",
-        "@eigen_archive//:eigen",
-        "@org_tensorflow//tensorflow/lite:framework",
-    ],
-    alwayslink = 1,
-)
-
-cc_library(
-    name = "lift_2d_frame_annotation_to_3d_calculator",
-    srcs = ["lift_2d_frame_annotation_to_3d_calculator.cc"],
-    visibility = ["//visibility:public"],
-    deps = [
-        ":annotation_cc_proto",
-        ":belief_decoder_config_cc_proto",
-        ":decoder",
-        ":lift_2d_frame_annotation_to_3d_calculator_cc_proto",
-        ":tensor_util",
-        ":tflite_tensors_to_objects_calculator_cc_proto",
-        "//mediapipe/framework:calculator_framework",
-        "//mediapipe/framework/deps:file_path",
-        "//mediapipe/framework/formats:detection_cc_proto",
-        "//mediapipe/framework/port:opencv_core",
-        "//mediapipe/framework/port:ret_check",
-        "@com_google_absl//absl/memory",
-        "@com_google_absl//absl/strings:str_format",
-        "@com_google_absl//absl/types:span",
-        "@eigen_archive//:eigen",
-        "@org_tensorflow//tensorflow/lite:framework",
-    ],
-    alwayslink = 1,
-)
-
 cc_library(
     name = "annotations_to_model_matrices_calculator",
     srcs = ["annotations_to_model_matrices_calculator.cc"],
     visibility = ["//visibility:public"],
     deps = [
-        ":annotation_cc_proto",
         ":annotations_to_model_matrices_calculator_cc_proto",
-        ":box",
         ":model_matrix_cc_proto",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework:calculator_options_cc_proto",
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework/port:status",
+        "//mediapipe/modules/objectron/calculators:annotation_cc_proto",
+        "//mediapipe/modules/objectron/calculators:box",
         "//mediapipe/util:color_cc_proto",
         "@com_google_absl//absl/memory",
         "@com_google_absl//absl/strings",
@@ -347,11 +99,11 @@ cc_library(
     srcs = ["annotations_to_render_data_calculator.cc"],
     visibility = ["//visibility:public"],
     deps = [
-        ":annotation_cc_proto",
         ":annotations_to_render_data_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework:calculator_options_cc_proto",
         "//mediapipe/framework/port:ret_check",
+        "//mediapipe/modules/objectron/calculators:annotation_cc_proto",
         "//mediapipe/util:color_cc_proto",
         "//mediapipe/util:render_data_cc_proto",
         "@com_google_absl//absl/memory",
@@ -359,76 +111,3 @@ cc_library(
     ],
     alwayslink = 1,
 )
-
-cc_library(
-    name = "frame_annotation_to_rect_calculator",
-    srcs = ["frame_annotation_to_rect_calculator.cc"],
-    deps = [
-        ":annotation_cc_proto",
-        ":box",
-        ":frame_annotation_to_rect_calculator_cc_proto",
-        "//mediapipe/framework:calculator_framework",
-        "//mediapipe/framework/formats:rect_cc_proto",
-        "//mediapipe/framework/port:ret_check",
-        "//mediapipe/framework/port:status",
-        "@com_google_absl//absl/memory",
-        "@eigen_archive//:eigen",
-    ],
-    alwayslink = 1,
-)
-
-cc_library(
-    name = "landmarks_to_frame_annotation_calculator",
-    srcs = ["landmarks_to_frame_annotation_calculator.cc"],
-    deps = [
-        ":annotation_cc_proto",
-        "//mediapipe/framework:calculator_framework",
-        "//mediapipe/framework/formats:landmark_cc_proto",
-        "//mediapipe/framework/port:ret_check",
-        "//mediapipe/framework/port:status",
-        "@com_google_absl//absl/memory",
-    ],
-    alwayslink = 1,
-)
-
-cc_library(
-    name = "filter_detection_calculator",
-    srcs = ["filter_detection_calculator.cc"],
-    deps = [
-        ":filter_detection_calculator_cc_proto",
-        "//mediapipe/framework:calculator_framework",
-        "//mediapipe/framework/formats:detection_cc_proto",
-        "//mediapipe/framework/formats:location_data_cc_proto",
-        "//mediapipe/framework/port:logging",
-        "//mediapipe/framework/port:map_util",
-        "//mediapipe/framework/port:re2",
-        "//mediapipe/framework/port:status",
-        "@com_google_absl//absl/container:node_hash_set",
-        "@com_google_absl//absl/strings",
-    ],
-    alwayslink = 1,
-)
-
-cc_test(
-    name = "box_util_test",
-    srcs = ["box_util_test.cc"],
-    deps = [
-        ":box_util",
-        "//mediapipe/framework/port:gtest_main",
-        "//mediapipe/framework/port:opencv_core",
-        "//mediapipe/util/tracking:box_tracker_cc_proto",
-    ],
-)
-
-cc_test(
-    name = "frame_annotation_tracker_test",
-    srcs = ["frame_annotation_tracker_test.cc"],
-    deps = [
-        ":annotation_cc_proto",
-        ":frame_annotation_tracker",
-        "//mediapipe/framework/port:gtest_main",
-        "//mediapipe/framework/port:logging",
-        "//mediapipe/util/tracking:box_tracker_cc_proto",
-        "@com_google_absl//absl/container:flat_hash_set",
-    ],
-)

mediapipe/graphs/object_detection_3d/calculators/a_r_capture_metadata.proto

@@ -1,551 +0,0 @@
-// Copyright 2020 The MediaPipe Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-syntax = "proto2";
-
-package mediapipe;
-
-// Info about the camera characteristics used to capture images and depth data.
-// See developer.apple.com/documentation/avfoundation/avcameracalibrationdata
-// for more information.
-message AVCameraCalibrationData {
-  // 3x3 row-major matrix relating a camera's internal properties to an ideal
-  // pinhole-camera model.
-  // See
-  // developer.apple.com/documentation/avfoundation/avcameracalibrationdata/2881135-intrinsicmatrix
-  // for detailed usage information.
-  repeated float intrinsic_matrix = 1 [packed = true];
-
-  // The image dimensions to which the intrinsic_matrix values are relative.
-  optional float intrinsic_matrix_reference_dimension_width = 2;
-  optional float intrinsic_matrix_reference_dimension_height = 3;
-
-  // 3x4 row-major matrix relating a camera's position and orientation to a
-  // world or scene coordinate system. Consists of a unitless 3x3 rotation
-  // matrix (R) on the left and a translation (t) 3x1 vector on the right. The
-  // translation vector's units are millimeters. For example:
-  //
-  //            |r1,1  r2,1  r3,1 | t1|
-  //  [R | t] = |r1,2  r2,2  r3,2 | t2|
-  //            |r1,3  r2,3  r3,3 | t3|
-  //
-  //  is stored as [r11, r21, r31, t1, r12, r22, r32, t2, ...]
-  //
-  // See
-  // developer.apple.com/documentation/avfoundation/avcameracalibrationdata/2881130-extrinsicmatrix?language=objc
-  // for more information.
-  repeated float extrinsic_matrix = 4 [packed = true];
-
-  // The size, in millimeters, of one image pixel.
-  optional float pixel_size = 5;
-
-  // A list of floating-point values describing radial distortions imparted by
-  // the camera lens, for use in rectifying camera images.
-  // See
-  // developer.apple.com/documentation/avfoundation/avcameracalibrationdata/2881129-lensdistortionlookuptable?language=objc
-  // for more information.
-  repeated float lens_distortion_lookup_values = 6 [packed = true];
-
-  // A list of floating-point values describing radial distortions for use in
-  // reapplying camera geometry to a rectified image.
-  // See
-  // developer.apple.com/documentation/avfoundation/avcameracalibrationdata/2881132-inverselensdistortionlookuptable?language=objc
-  // for more information.
-  repeated float inverse_lens_distortion_lookup_values = 7 [packed = true];
-
-  // The offset of the distortion center of the camera lens from the top-left
-  // corner of the image.
-  // See
-  // developer.apple.com/documentation/avfoundation/avcameracalibrationdata/2881131-lensdistortioncenter?language=objc
-  // for more information.
-  optional float lens_distortion_center_x = 8;
-  optional float lens_distortion_center_y = 9;
-}
-
-// Container for depth data information.
-// See developer.apple.com/documentation/avfoundation/avdepthdata for more info.
-message AVDepthData {
-  // PNG representation of the grayscale depth data map. See discussion about
-  // depth_data_map_original_minimum_value, below, for information about how
-  // to interpret the pixel values.
-  optional bytes depth_data_map = 1;
-
-  // Pixel format type of the original captured depth data.
-  // See
-  // developer.apple.com/documentation/corevideo/1563591-pixel_format_identifiers?language=objc
-  // for the complete list of possible pixel format types. This value represents
-  // a string for the associated OSType/FourCharCode.
-  optional string depth_data_type = 2;
-
-  // Indicates the general accuracy of the depth_data_map.
-  // See developer.apple.com/documentation/avfoundation/avdepthdataaccuracy for
-  // more information.
-  enum Accuracy {
-    UNDEFINED_ACCURACY = 0;
-    // Values in the depth map are usable for foreground/background separation
-    // but are not absolutely accurate in the physical world.
-    RELATIVE = 1;
-    // Values in the depth map are absolutely accurate in the physical world.
-    ABSOLUTE = 2;
-  }
-  optional Accuracy depth_data_accuracy = 3 [default = RELATIVE];
-
-  // Indicates whether the depth_data_map contains temporally smoothed data.
-  optional bool depth_data_filtered = 4;
-
-  // Quality of the depth_data_map.
-  enum Quality {
-    UNDEFINED_QUALITY = 0;
-    HIGH = 1;
-    LOW = 2;
-  }
-  optional Quality depth_data_quality = 5;
-
-  // Associated calibration data for the depth_data_map.
-  optional AVCameraCalibrationData camera_calibration_data = 6;
-
-  // The original range of values expressed by the depth_data_map, before
-  // grayscale normalization. For example, if the minimum and maximum values
-  // indicate a range of [0.5, 2.2], and the depth_data_type value indicates
-  // it was a depth map, then white pixels (255, 255, 255) will map to 0.5 and
-  // black pixels (0, 0, 0) will map to 2.2 with the grayscale range linearly
-  // interpolated inbetween. Conversely, if the depth_data_type value indicates
-  // it was a disparity map, then white pixels will map to 2.2 and black pixels
-  // will map to 0.5.
-  optional float depth_data_map_original_minimum_value = 7;
-  optional float depth_data_map_original_maximum_value = 8;
-
-  // The width of the depth buffer map.
-  optional int32 depth_data_map_width = 9;
-
-  // The height of the depth buffer map.
-  optional int32 depth_data_map_height = 10;
-
-  // The row-major flattened array of the depth buffer map pixels. This will be
-  // either a float32 or float16 byte array, depending on 'depth_data_type'.
-  optional bytes depth_data_map_raw_values = 11;
-}
-
-// Estimated scene lighting information associated with a captured video frame.
-// See developer.apple.com/documentation/arkit/arlightestimate for more info.
-message ARLightEstimate {
-  // The estimated intensity, in lumens, of ambient light throughout the scene.
-  optional double ambient_intensity = 1;
-
-  // The estimated color temperature, in degrees Kelvin, of ambient light
-  // throughout the scene.
-  optional double ambient_color_temperature = 2;
-
-  // Data describing the estimated lighting environment in all directions.
-  // Second-level spherical harmonics in separate red, green, and blue data
-  // planes. Thus, this buffer contains 3 sets of 9 coefficients, or a total of
-  // 27 values.
-  // See
-  // https://developer.apple.com/documentation/arkit/ardirectionallightestimate/2928222-sphericalharmonicscoefficients?language=objc
-  // for more information.
-  repeated float spherical_harmonics_coefficients = 3 [packed = true];
-
-  message DirectionVector {
-    optional float x = 1;
-    optional float y = 2;
-    optional float z = 3;
-  }
-  // A vector indicating the orientation of the strongest directional light
-  // source, normalized in the world-coordinate space.
-  // See
-  // https://developer.apple.com/documentation/arkit/ardirectionallightestimate/2928221-primarylightdirection?language=objc
-  // for more information;
-  optional DirectionVector primary_light_direction = 4;
-
-  // The estimated intensity, in lumens, of the strongest directional light
-  // source in the scene.
-  // See
-  // https://developer.apple.com/documentation/arkit/ardirectionallightestimate/2928219-primarylightintensity?language=objc
-  // for more information.
-  optional float primary_light_intensity = 5;
-}
-
-// Information about the camera position and imaging characteristics for a
-// captured video frame.
-// See developer.apple.com/documentation/arkit/arcamera for more information.
-message ARCamera {
-  // The general quality of position tracking available when the camera captured
-  // a frame.
-  enum TrackingState {
-    UNDEFINED_TRACKING_STATE = 0;
-    // Camera position tracking is not available.
-    UNAVAILABLE = 1;
-    // Tracking is available, but the quality of results is questionable.
-    LIMITED = 2;
-    // Camera position tracking is providing optimal results.
-    NORMAL = 3;
-  }
-  optional TrackingState tracking_state = 1 [default = UNAVAILABLE];
-
-  // A possible diagnosis for limited position tracking quality as of when the
-  // frame was captured.
-  enum TrackingStateReason {
-    UNDEFINED_TRACKING_STATE_REASON = 0;
-    // The current tracking state is not limited.
-    NONE = 1;
-    // Not yet enough camera or motion data to provide tracking information.
-    INITIALIZING = 2;
-    // The device is moving too fast for accurate image-based position tracking.
-    EXCESSIVE_MOTION = 3;
-    // Not enough distinguishable features for image-based position tracking.
-    INSUFFICIENT_FEATURES = 4;
-    // Tracking is limited due to a relocalization in progress.
-    RELOCALIZING = 5;
-  }
-  optional TrackingStateReason tracking_state_reason = 2 [default = NONE];
-
-  // 4x4 row-major matrix expressing position and orientation of the camera in
-  // world coordinate space.
-  // See developer.apple.com/documentation/arkit/arcamera/2866108-transform for
-  // more information.
-  repeated float transform = 3 [packed = true];
-
-  // The orientation of the camera, expressed as roll, pitch, and yaw values.
-  message EulerAngles {
-    optional float roll = 1;
-    optional float pitch = 2;
-    optional float yaw = 3;
-  }
-  optional EulerAngles euler_angles = 4;
-
-  // The width and height, in pixels, of the captured camera image.
-  optional int32 image_resolution_width = 5;
-  optional int32 image_resolution_height = 6;
-
-  // 3x3 row-major matrix that converts between the 2D camera plane and 3D world
-  // coordinate space.
-  // See developer.apple.com/documentation/arkit/arcamera/2875730-intrinsics for
-  // usage information.
-  repeated float intrinsics = 7 [packed = true];
-
-  // 4x4 row-major transform matrix appropriate for rendering 3D content to
-  // match the image captured by the camera.
-  // See
-  // developer.apple.com/documentation/arkit/arcamera/2887458-projectionmatrix
-  // for usage information.
-  repeated float projection_matrix = 8 [packed = true];
-
-  // 4x4 row-major transform matrix appropriate for converting from world-space
-  // to camera space. Relativized for the captured_image orientation (i.e.
-  // UILandscapeOrientationRight).
-  // See
-  // https://developer.apple.com/documentation/arkit/arcamera/2921672-viewmatrixfororientation?language=objc
-  // for more information.
-  repeated float view_matrix = 9 [packed = true];
-}
-
-// Container for a 3D mesh describing face topology.
-message ARFaceGeometry {
-  // Each vertex represents a 3D point in the face mesh, in the face coordinate
-  // space.
-  // See developer.apple.com/documentation/arkit/arfacegeometry/2928201-vertices
-  // for more information.
-  message Vertex {
-    optional float x = 1;
-    optional float y = 2;
-    optional float z = 3;
-  }
-  repeated Vertex vertices = 1;
-
-  // The number of elements in the vertices list.
-  optional int32 vertex_count = 2;
-
-  // Each texture coordinate represents UV texture coordinates for the vertex at
-  // the corresponding index in the vertices buffer.
-  // See
-  // developer.apple.com/documentation/arkit/arfacegeometry/2928203-texturecoordinates
-  // for more information.
-  message TextureCoordinate {
-    optional float u = 1;
-    optional float v = 2;
-  }
-  repeated TextureCoordinate texture_coordinates = 3;
-
-  // The number of elements in the texture_coordinates list.
-  optional int32 texture_coordinate_count = 4;
-
-  // Each integer value in this ordered list represents an index into the
-  // vertices and texture_coordinates lists. Each set of three indices
-  // identifies the vertices comprising a single triangle in the mesh. Each set
-  // of three indices forms a triangle, so the number of indices in the
-  // triangle_indices buffer is three times the triangle_count value.
-  // See
-  // developer.apple.com/documentation/arkit/arfacegeometry/2928199-triangleindices
-  // for more information.
-  repeated int32 triangle_indices = 5 [packed = true];
-
-  // The number of triangles described by the triangle_indices buffer.
-  // See
-  // developer.apple.com/documentation/arkit/arfacegeometry/2928207-trianglecount
-  // for more information.
-  optional int32 triangle_count = 6;
-}
-
-// Contains a list of blend shape entries wherein each item maps a specific
-// blend shape location to its associated coefficient.
-message ARBlendShapeMap {
-  message MapEntry {
-    // Identifier for the specific facial feature.
-    // See developer.apple.com/documentation/arkit/arblendshapelocation for a
-    // complete list of identifiers.
-    optional string blend_shape_location = 1;
-
-    // Indicates the current position of the feature relative to its neutral
-    // configuration, ranging from 0.0 (neutral) to 1.0 (maximum movement).
-    optional float blend_shape_coefficient = 2;
-  }
-  repeated MapEntry entries = 1;
-}
-
-// Information about the pose, topology, and expression of a detected face.
-// See developer.apple.com/documentation/arkit/arfaceanchor for more info.
-message ARFaceAnchor {
-  // A coarse triangle mesh representing the topology of the detected face.
-  optional ARFaceGeometry geometry = 1;
-
-  // A map of named coefficients representing the detected facial expression in
-  // terms of the movement of specific facial features.
-  optional ARBlendShapeMap blend_shapes = 2;
-
-  // 4x4 row-major matrix encoding the position, orientation, and scale of the
-  // anchor relative to the world coordinate space.
-  // See
-  // https://developer.apple.com/documentation/arkit/aranchor/2867981-transform?language=objc
-  // for more information.
-  repeated float transform = 3;
-
-  // Indicates whether the anchor's transform is valid. Frames that have a face
-  // anchor with this value set to NO should probably be ignored.
-  optional bool is_tracked = 4;
-}
-
-// Container for a 3D mesh.
-message ARPlaneGeometry {
-  message Vertex {
-    optional float x = 1;
-    optional float y = 2;
-    optional float z = 3;
-  }
-
-  // Each texture coordinate represents UV texture coordinates for the vertex at
-  // the corresponding index in the vertices buffer.
-  // See
-  // https://developer.apple.com/documentation/arkit/arfacegeometry/2928203-texturecoordinates
-  // for more information.
-  message TextureCoordinate {
-    optional float u = 1;
-    optional float v = 2;
-  }
-
-  // A buffer of vertex positions for each point in the plane mesh.
-  repeated Vertex vertices = 1;
-
-  // The number of elements in the vertices buffer.
-  optional int32 vertex_count = 2;
-
-  // A buffer of texture coordinate values for each point in the plane mesh.
-  repeated TextureCoordinate texture_coordinates = 3;
-
-  // The number of elements in the texture_coordinates buffer.
-  optional int32 texture_coordinate_count = 4;
-
-  // Each integer value in this ordered list represents an index into the
-  // vertices and texture_coordinates lists. Each set of three indices
-  // identifies the vertices comprising a single triangle in the mesh. Each set
-  // of three indices forms a triangle, so the number of indices in the
-  // triangle_indices buffer is three times the triangle_count value.
-  // See
-  // https://developer.apple.com/documentation/arkit/arplanegeometry/2941051-triangleindices
-  // for more information.
-  repeated int32 triangle_indices = 5 [packed = true];
-
-  // Each set of three indices forms a triangle, so the number of indices in the
-  // triangle_indices buffer is three times the triangle_count value.
-  // See
-  // https://developer.apple.com/documentation/arkit/arplanegeometry/2941058-trianglecount
-  // for more information.
-  optional int32 triangle_count = 6;
-
-  // Each value in this buffer represents the position of a vertex along the
-  // boundary polygon of the estimated plane. The owning plane anchor's
-  // transform matrix defines the coordinate system for these points.
-  // See
-  // https://developer.apple.com/documentation/arkit/arplanegeometry/2941052-boundaryvertices
-  // for more information.
-  repeated Vertex boundary_vertices = 7;
-
-  // The number of elements in the boundary_vertices buffer.
-  optional int32 boundary_vertex_count = 8;
-}
-
-// Information about the position and orientation of a real-world flat surface.
-// See https://developer.apple.com/documentation/arkit/arplaneanchor for more
-// information.
-message ARPlaneAnchor {
-  enum Alignment {
-    UNDEFINED = 0;
-    // The plane is perpendicular to gravity.
-    HORIZONTAL = 1;
-    // The plane is parallel to gravity.
-    VERTICAL = 2;
-  }
-
-  // Wrapper for a 3D point / vector within the plane. See extent and center
-  // values for more information.
-  message PlaneVector {
-    optional float x = 1;
-    optional float y = 2;
-    optional float z = 3;
-  }
-
-  enum PlaneClassification {
-    NONE = 0;
-    WALL = 1;
-    FLOOR = 2;
-    CEILING = 3;
-    TABLE = 4;
-    SEAT = 5;
-  }
-
-  // The classification status for the plane.
-  enum PlaneClassificationStatus {
-    // The classfication process for the plane anchor has completed but the
-    // result is inconclusive.
-    UNKNOWN = 0;
-    // No classication information can be provided (set on error or if the
-    // device does not support plane classification).
-    UNAVAILABLE = 1;
-    // The classification process has not completed.
-    UNDETERMINED = 2;
-    // The classfication process for the plane anchor has completed.
-    KNOWN = 3;
-  }
-
-  // The ID of the plane.
-  optional string identifier = 1;
-
-  // 4x4 row-major matrix encoding the position, orientation, and scale of the
-  // anchor relative to the world coordinate space.
-  // See
-  // https://developer.apple.com/documentation/arkit/aranchor/2867981-transform
-  // for more information.
-  repeated float transform = 2;
-
-  // The general orientation of the detected plane with respect to gravity.
-  optional Alignment alignment = 3;
-
-  // A coarse triangle mesh representing the general shape of the detected
-  // plane.
-  optional ARPlaneGeometry geometry = 4;
-
-  // The center point of the plane relative to its anchor position.
-  // Although the type of this property is a 3D vector, a plane anchor is always
-  // two-dimensional, and is always positioned in only the x and z directions
-  // relative to its transform position. (That is, the y-component of this
-  // vector is always zero.)
-  // See
-  // https://developer.apple.com/documentation/arkit/arplaneanchor/2882056-center
-  // for more information.
-  optional PlaneVector center = 5;
-
-  // The estimated width and length of the detected plane.
-  // See
-  // https://developer.apple.com/documentation/arkit/arplaneanchor/2882055-extent
-  // for more information.
-  optional PlaneVector extent = 6;
-
-  // A Boolean value that indicates whether plane classification is available on
-  // the current device. On devices without plane classification support, all
-  // plane anchors report a classification value of NONE
-  // and a classification_status value of UNAVAILABLE.
-  optional bool classification_supported = 7;
-
-  // A general characterization of what kind of real-world surface the plane
-  // anchor represents.
-  // See
-  // https://developer.apple.com/documentation/arkit/arplaneanchor/2990936-classification
-  // for more information.
-  optional PlaneClassification classification = 8;
-
-  // The current state of ARKit's process for classifying the plane anchor.
-  // When this property's value is KNOWN, the classification property represents
-  // ARKit's characterization of the real-world surface corresponding to the
-  // plane anchor.
-  // See
-  // https://developer.apple.com/documentation/arkit/arplaneanchor/2990937-classificationstatus
-  // for more information.
-  optional PlaneClassificationStatus classification_status = 9;
-}
-
-// A collection of points in the world coordinate space.
-// See https://developer.apple.com/documentation/arkit/arpointcloud for more
-// information.
-message ARPointCloud {
-  message Point {
-    optional float x = 1;
-    optional float y = 2;
-    optional float z = 3;
-  }
-
-  // The number of points in the cloud.
-  optional int32 count = 1;
-
-  // The list of detected points.
-  repeated Point point = 2;
-
-  // A list of unique identifiers corresponding to detected feature points.
-  // Each identifier in this list corresponds to the point at the same index
-  // in the points array.
-  repeated int64 identifier = 3 [packed = true];
-}
-
-// Video image and face position tracking information.
-// See developer.apple.com/documentation/arkit/arframe for more information.
-message ARFrame {
-  // The timestamp for the frame.
-  optional double timestamp = 1;
-
-  // The depth data associated with the frame. Not all frames have depth data.
-  optional AVDepthData depth_data = 2;
-
-  // The depth data object timestamp associated with the frame. May differ from
-  // the frame timestamp value. Is only set when the frame has depth_data.
-  optional double depth_data_timestamp = 3;
-
-  // Camera information associated with the frame.
-  optional ARCamera camera = 4;
-
-  // Light information associated with the frame.
-  optional ARLightEstimate light_estimate = 5;
-
-  // Face anchor information associated with the frame. Not all frames have an
-  // active face anchor.
-  optional ARFaceAnchor face_anchor = 6;
-
-  // Plane anchors associated with the frame. Not all frames have a plane
-  // anchor. Plane anchors and face anchors are mutually exclusive.
-  repeated ARPlaneAnchor plane_anchor = 7;
-
-  // The current intermediate results of the scene analysis used to perform
-  // world tracking.
-  // See
-  // https://developer.apple.com/documentation/arkit/arframe/2887449-rawfeaturepoints
-  // for more information.
-  optional ARPointCloud raw_feature_points = 8;
-}

mediapipe/graphs/object_detection_3d/calculators/annotation_data.proto

@@ -1,92 +0,0 @@
-// Copyright 2020 The MediaPipe Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-syntax = "proto3";
-
-package mediapipe;
-
-import "mediapipe/graphs/object_detection_3d/calculators/a_r_capture_metadata.proto";
-import "mediapipe/graphs/object_detection_3d/calculators/object.proto";
-
-// Projection of a 3D point on an image, and its metric depth.
-message NormalizedPoint2D {
-  // x-y position of the 2d keypoint in the image coordinate system.
-  // u,v \in [0, 1], where top left corner is (0, 0) and the bottom-right corner
-  // is (1, 1).
-  float x = 1;
-  float y = 2;
-
-  // The depth of the point in the camera coordinate system (in meters).
-  float depth = 3;
-}
-
-// The 3D point in the camera coordinate system, the scales are in meters.
-message Point3D {
-  float x = 1;
-  float y = 2;
-  float z = 3;
-}
-
-message AnnotatedKeyPoint {
-  int32 id = 1;
-  Point3D point_3d = 2;
-  NormalizedPoint2D point_2d = 3;
-}
-
-message ObjectAnnotation {
-  // Reference to the object identifier in ObjectInstance.
-  int32 object_id = 1;
-
-  // For each objects, list all the annotated keypoints here.
-  // E.g. for bounding-boxes, we have 8 keypoints, hands = 21 keypoints, etc.
-  // These normalized points are the projection of the Object's 3D keypoint
-  // on the current frame's camera poses.
-  repeated AnnotatedKeyPoint keypoints = 2;
-
-  // Visibiity of this annotation in a frame.
-  float visibility = 3;
-}
-
-message FrameAnnotation {
-  // Unique frame id, corresponds to images.
-  int32 frame_id = 1;
-
-  // List of the annotated objects in this frame. Depending on how many object
-  // are observable in this frame, we might have non or as much as
-  // sequence.objects_size() annotations.
-  repeated ObjectAnnotation annotations = 2;
-
-  // Information about the camera transformation (in the world coordinate) and
-  // imaging characteristics for a captured video frame.
-  ARCamera camera = 3;
-
-  // The timestamp for the frame.
-  double timestamp = 4;
-
-  // Plane center and normal in camera frame.
-  repeated float plane_center = 5;
-  repeated float plane_normal = 6;
-}
-
-// The sequence protocol contains the annotation data for the entire video clip.
-message Sequence {
-  // List of all the annotated 3D objects in this sequence in the world
-  // Coordinate system. Given the camera poses of each frame (also in the
-  // world-coordinate) these objects bounding boxes can be projected to each
-  // frame to get the per-frame annotation (i.e. image_annotation below).
-  repeated Object objects = 1;
-
-  // List of annotated data per each frame in sequence + frame information.
-  repeated FrameAnnotation frame_annotations = 2;
-}

mediapipe/graphs/object_detection_3d/calculators/annotations_to_model_matrices_calculator.cc

@@ -24,10 +24,10 @@
 #include "mediapipe/framework/calculator_options.pb.h"
 #include "mediapipe/framework/port/ret_check.h"
 #include "mediapipe/framework/port/status.h"
-#include "mediapipe/graphs/object_detection_3d/calculators/annotation_data.pb.h"
 #include "mediapipe/graphs/object_detection_3d/calculators/annotations_to_model_matrices_calculator.pb.h"
-#include "mediapipe/graphs/object_detection_3d/calculators/box.h"
 #include "mediapipe/graphs/object_detection_3d/calculators/model_matrix.pb.h"
+#include "mediapipe/modules/objectron/calculators/annotation_data.pb.h"
+#include "mediapipe/modules/objectron/calculators/box.h"
 #include "mediapipe/util/color.pb.h"
 
 namespace mediapipe {
@@ -66,14 +66,14 @@ class AnnotationsToModelMatricesCalculator : public CalculatorBase {
   AnnotationsToModelMatricesCalculator& operator=(
       const AnnotationsToModelMatricesCalculator&) = delete;
 
-  static ::mediapipe::Status GetContract(CalculatorContract* cc);
+  static mediapipe::Status GetContract(CalculatorContract* cc);
 
-  ::mediapipe::Status Open(CalculatorContext* cc) override;
+  mediapipe::Status Open(CalculatorContext* cc) override;
 
-  ::mediapipe::Status Process(CalculatorContext* cc) override;
+  mediapipe::Status Process(CalculatorContext* cc) override;
 
  private:
-  ::mediapipe::Status GetModelMatricesForAnnotations(
+  mediapipe::Status GetModelMatricesForAnnotations(
       const FrameAnnotation& annotations,
       TimedModelMatrixProtoList* model_matrix_list);
 
@@ -83,7 +83,7 @@ class AnnotationsToModelMatricesCalculator : public CalculatorBase {
 };
 REGISTER_CALCULATOR(AnnotationsToModelMatricesCalculator);
 
-::mediapipe::Status AnnotationsToModelMatricesCalculator::GetContract(
+mediapipe::Status AnnotationsToModelMatricesCalculator::GetContract(
     CalculatorContract* cc) {
   RET_CHECK(cc->Inputs().HasTag(kAnnotationTag)) << "No input stream found.";
   if (cc->Inputs().HasTag(kAnnotationTag)) {
@@ -101,10 +101,10 @@ REGISTER_CALCULATOR(AnnotationsToModelMatricesCalculator);
   if (cc->InputSidePackets().HasTag("MODEL_TRANSFORMATION")) {
     cc->InputSidePackets().Tag("MODEL_TRANSFORMATION").Set<float[]>();
   }
-  return ::mediapipe::OkStatus();
+  return mediapipe::OkStatus();
 }
 
-::mediapipe::Status AnnotationsToModelMatricesCalculator::Open(
+mediapipe::Status AnnotationsToModelMatricesCalculator::Open(
     CalculatorContext* cc) {
   RET_CHECK(cc->Inputs().HasTag(kAnnotationTag));
 
@@ -131,10 +131,10 @@ REGISTER_CALCULATOR(AnnotationsToModelMatricesCalculator);
     model_transformation_.setIdentity();
   }
 
-  return ::mediapipe::OkStatus();
+  return mediapipe::OkStatus();
 }
 
-::mediapipe::Status AnnotationsToModelMatricesCalculator::Process(
+mediapipe::Status AnnotationsToModelMatricesCalculator::Process(
     CalculatorContext* cc) {
   auto model_matrices = std::make_unique<TimedModelMatrixProtoList>();
 
@@ -142,22 +142,21 @@ REGISTER_CALCULATOR(AnnotationsToModelMatricesCalculator);
       cc->Inputs().Tag(kAnnotationTag).Get<FrameAnnotation>();
 
   if (!GetModelMatricesForAnnotations(annotations, model_matrices.get()).ok()) {
-    return ::mediapipe::InvalidArgumentError(
-        "Error in GetModelMatricesForBoxes");
+    return mediapipe::InvalidArgumentError("Error in GetModelMatricesForBoxes");
   }
   cc->Outputs()
       .Tag(kModelMatricesTag)
       .Add(model_matrices.release(), cc->InputTimestamp());
 
-  return ::mediapipe::OkStatus();
+  return mediapipe::OkStatus();
 }
 
-::mediapipe::Status
+mediapipe::Status
 AnnotationsToModelMatricesCalculator::GetModelMatricesForAnnotations(
     const FrameAnnotation& annotations,
     TimedModelMatrixProtoList* model_matrix_list) {
   if (model_matrix_list == nullptr) {
-    return ::mediapipe::InvalidArgumentError("model_matrix_list is nullptr");
+    return mediapipe::InvalidArgumentError("model_matrix_list is nullptr");
   }
   model_matrix_list->clear_model_matrix();
 
@@ -217,7 +216,7 @@ AnnotationsToModelMatricesCalculator::GetModelMatricesForAnnotations(
       }
     }
   }
-  return ::mediapipe::OkStatus();
+  return mediapipe::OkStatus();
 }
 
 }  // namespace mediapipe

mediapipe/graphs/object_detection_3d/calculators/annotations_to_render_data_calculator.cc

@@ -18,8 +18,8 @@
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/calculator_options.pb.h"
 #include "mediapipe/framework/port/ret_check.h"
-#include "mediapipe/graphs/object_detection_3d/calculators/annotation_data.pb.h"
 #include "mediapipe/graphs/object_detection_3d/calculators/annotations_to_render_data_calculator.pb.h"
+#include "mediapipe/modules/objectron/calculators/annotation_data.pb.h"
 #include "mediapipe/util/color.pb.h"
 #include "mediapipe/util/render_data.pb.h"
 
@@ -98,11 +98,11 @@ class AnnotationsToRenderDataCalculator : public CalculatorBase {
   AnnotationsToRenderDataCalculator& operator=(
       const AnnotationsToRenderDataCalculator&) = delete;
 
-  static ::mediapipe::Status GetContract(CalculatorContract* cc);
+  static mediapipe::Status GetContract(CalculatorContract* cc);
 
-  ::mediapipe::Status Open(CalculatorContext* cc) override;
+  mediapipe::Status Open(CalculatorContext* cc) override;
 
-  ::mediapipe::Status Process(CalculatorContext* cc) override;
+  mediapipe::Status Process(CalculatorContext* cc) override;
 
  private:
   static void SetRenderAnnotationColorThickness(
@@ -134,7 +134,7 @@ class AnnotationsToRenderDataCalculator : public CalculatorBase {
 };
 REGISTER_CALCULATOR(AnnotationsToRenderDataCalculator);
 
-::mediapipe::Status AnnotationsToRenderDataCalculator::GetContract(
+mediapipe::Status AnnotationsToRenderDataCalculator::GetContract(
     CalculatorContract* cc) {
   RET_CHECK(cc->Inputs().HasTag(kAnnotationTag)) << "No input stream found.";
   if (cc->Inputs().HasTag(kAnnotationTag)) {
@@ -142,18 +142,18 @@ REGISTER_CALCULATOR(AnnotationsToRenderDataCalculator);
   }
   cc->Outputs().Tag(kRenderDataTag).Set<RenderData>();
 
-  return ::mediapipe::OkStatus();
+  return mediapipe::OkStatus();
 }
 
-::mediapipe::Status AnnotationsToRenderDataCalculator::Open(
+mediapipe::Status AnnotationsToRenderDataCalculator::Open(
     CalculatorContext* cc) {
   cc->SetOffset(TimestampDiff(0));
   options_ = cc->Options<AnnotationsToRenderDataCalculatorOptions>();
 
-  return ::mediapipe::OkStatus();
+  return mediapipe::OkStatus();
 }
 
-::mediapipe::Status AnnotationsToRenderDataCalculator::Process(
+mediapipe::Status AnnotationsToRenderDataCalculator::Process(
     CalculatorContext* cc) {
   auto render_data = absl::make_unique<RenderData>();
   bool visualize_depth = options_.visualize_landmark_depth();
@@ -215,7 +215,7 @@ REGISTER_CALCULATOR(AnnotationsToRenderDataCalculator);
       .Tag(kRenderDataTag)
       .Add(render_data.release(), cc->InputTimestamp());
 
-  return ::mediapipe::OkStatus();
+  return mediapipe::OkStatus();
 }
 
 void AnnotationsToRenderDataCalculator::AddConnectionToRenderData(

mediapipe/graphs/object_detection_3d/calculators/belief_decoder_config.proto

@@ -1,38 +0,0 @@
-// Copyright 2020 The MediaPipe Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-syntax = "proto2";
-
-package mediapipe;
-
-message BeliefDecoderConfig {
-  optional float heatmap_threshold = 1 [default = 0.9];
-  // Maximum distance in pixels between two local max heatmap values.
-  optional float local_max_distance = 2 [default = 10.0];
-  // Coefficient of offset_scale.
-  // offset_scale = offset_scale_coef * min(rows, cols).
-  // offset_scale is used to multiply the offset predictions from the network.
-  optional float offset_scale_coef = 3 [default = 0.5, deprecated = true];
-
-  // The radius for vertex voting. Use no voting if the radius is less than or
-  // euqal to 1. Example: 10.
-  optional int32 voting_radius = 4;
-
-  // The number of pixels to determine whether two points are the same.
-  // Example: 5 (voting_radius / 2).
-  optional int32 voting_allowance = 5;
-
-  // The threshold of beliefs, with which the points can vote. Example: 0.2.
-  optional float voting_threshold = 6;
-}

mediapipe/graphs/object_detection_3d/calculators/box.cc

@@ -1,255 +0,0 @@
-// Copyright 2020 The MediaPipe Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "mediapipe/graphs/object_detection_3d/calculators/box.h"
-
-#include "Eigen/src/Core/util/Constants.h"
-#include "mediapipe/framework/port/logging.h"
-
-namespace mediapipe {
-
-namespace {
-constexpr int kFrontFaceId = 4;
-constexpr int kTopFaceId = 2;
-constexpr int kNumKeypoints = 8 + 1;
-constexpr int kNumberOfAxis = 3;
-constexpr int kEdgesPerAxis = 4;
-
-}  // namespace
-
-Box::Box(const std::string& category)
-    : Model(kBoundingBox, kNumKeypoints, category),
-      bounding_box_(kNumKeypoints) {
-  transformation_.setIdentity();
-
-  scale_ << 0.1, 0.1, 0.1;
-
-  // The vertices are ordered according to the left-hand rule, so the normal
-  // vector of each face will point inward the box.
-  faces_.push_back({5, 6, 8, 7});  // +x on yz plane
-  faces_.push_back({1, 3, 4, 2});  // -x on yz plane
-
-  faces_.push_back({3, 7, 8, 4});  // +y on xz plane = top
-  faces_.push_back({1, 2, 6, 5});  // -y on xz plane
-
-  faces_.push_back({2, 4, 8, 6});  // +z on xy plane = front
-  faces_.push_back({1, 5, 7, 3});  // -z on xy plane
-
-  // Add the edges in the cube, they are sorted according to axis (x-y-z).
-  edges_.push_back({1, 5});
-  edges_.push_back({2, 6});
-  edges_.push_back({3, 7});
-  edges_.push_back({4, 8});
-
-  edges_.push_back({1, 3});
-  edges_.push_back({5, 7});
-  edges_.push_back({2, 4});
-  edges_.push_back({6, 8});
-
-  edges_.push_back({1, 2});
-  edges_.push_back({3, 4});
-  edges_.push_back({5, 6});
-  edges_.push_back({7, 8});
-  Update();
-}
-
-void Box::Update() {
-  // Compute the eight vertices of the bounding box from Box's parameters
-  auto w = scale_[0] / 2.f;
-  auto h = scale_[1] / 2.f;
-  auto d = scale_[2] / 2.f;
-
-  // Define the local coordinate system, w.r.t. the center of the boxs
-  bounding_box_[0] << 0., 0., 0.;
-  bounding_box_[1] << -w, -h, -d;
-  bounding_box_[2] << -w, -h, +d;
-  bounding_box_[3] << -w, +h, -d;
-  bounding_box_[4] << -w, +h, +d;
-  bounding_box_[5] << +w, -h, -d;
-  bounding_box_[6] << +w, -h, +d;
-  bounding_box_[7] << +w, +h, -d;
-  bounding_box_[8] << +w, +h, +d;
-
-  // Convert to world coordinate system
-  for (int i = 0; i < kNumKeypoints; ++i) {
-    bounding_box_[i] =
-        transformation_.topLeftCorner<3, 3>() * bounding_box_[i] +
-        transformation_.col(3).head<3>();
-  }
-}
-
-void Box::Adjust(const std::vector<float>& variables) {
-  Eigen::Vector3f translation;
-  translation << variables[0], variables[1], variables[2];
-  SetTranslation(translation);
-
-  const float roll = variables[3];
-  const float pitch = variables[4];
-  const float yaw = variables[5];
-  SetRotation(roll, pitch, yaw);
-
-  Eigen::Vector3f scale;
-  scale << variables[6], variables[7], variables[8];
-
-  SetScale(scale);
-  Update();
-}
-
-float* Box::GetVertex(size_t vertex_id) {
-  CHECK_LT(vertex_id, kNumKeypoints);
-  return bounding_box_[vertex_id].data();
-}
-
-const float* Box::GetVertex(size_t vertex_id) const {
-  CHECK_LT(vertex_id, kNumKeypoints);
-  return bounding_box_[vertex_id].data();
-}
-
-bool Box::InsideTest(const Eigen::Vector3f& point, int check_axis) const {
-  const float* v0 = GetVertex(1);
-  const float* v1 = GetVertex(2);
-  const float* v2 = GetVertex(3);
-  const float* v4 = GetVertex(5);
-
-  switch (check_axis) {
-    case 1:
-      return (v0[0] <= point[0] && point[0] <= v1[0]);  // X-axis
-    case 2:
-      return (v0[1] <= point[1] && point[1] <= v2[1]);  // Y-axis
-    case 3:
-      return (v0[2] <= point[2] && point[2] <= v4[2]);  // Z-axis
-    default:
-      return false;
-  }
-}
-
-void Box::Deserialize(const Object& obj) {
-  CHECK_EQ(obj.keypoints_size(), kNumKeypoints);
-  Model::Deserialize(obj);
-}
-
-void Box::Serialize(Object* obj) {
-  Model::Serialize(obj);
-  obj->set_type(Object::BOUNDING_BOX);
-  std::vector<Vector3f> local_bounding_box(9);
-  // Define the local coordinate system, w.r.t. the center of the boxs
-  local_bounding_box[0] << 0., 0., 0.;
-  local_bounding_box[1] << -0.5, -0.5, -0.5;
-  local_bounding_box[2] << -0.5, -0.5, +0.5;
-  local_bounding_box[3] << -0.5, +0.5, -0.5;
-  local_bounding_box[4] << -0.5, +0.5, +0.5;
-  local_bounding_box[5] << +0.5, -0.5, -0.5;
-  local_bounding_box[6] << +0.5, -0.5, +0.5;
-  local_bounding_box[7] << +0.5, +0.5, -0.5;
-  local_bounding_box[8] << +0.5, +0.5, +0.5;
-  for (int i = 0; i < kNumKeypoints; ++i) {
-    KeyPoint* keypoint = obj->add_keypoints();
-    keypoint->set_x(local_bounding_box[i][0]);
-    keypoint->set_y(local_bounding_box[i][1]);
-    keypoint->set_z(local_bounding_box[i][2]);
-    keypoint->set_confidence_radius(0.);
-  }
-}
-
-const Face& Box::GetFrontFace() const { return faces_[kFrontFaceId]; }
-
-const Face& Box::GetTopFace() const { return faces_[kTopFaceId]; }
-
-std::pair<Vector3f, Vector3f> Box::GetGroundPlane() const {
-  const Vector3f gravity = Vector3f(0., 1., 0.);
-  int ground_plane_id = 0;
-  float ground_plane_error = 10.0;
-
-  auto get_face_center = [&](const Face& face) {
-    Vector3f center = Vector3f::Zero();
-    for (const int vertex_id : face) {
-      center += Map<const Vector3f>(GetVertex(vertex_id));
-    }
-    center /= face.size();
-    return center;
-  };
-
-  auto get_face_normal = [&](const Face& face, const Vector3f& center) {
-    Vector3f v1 = Map<const Vector3f>(GetVertex(face[0])) - center;
-    Vector3f v2 = Map<const Vector3f>(GetVertex(face[1])) - center;
-    Vector3f normal = v1.cross(v2);
-    return normal;
-  };
-
-  // The ground plane is defined as a plane aligned with gravity.
-  // gravity is the (0, 1, 0) vector in the world coordinate system.
-  const auto& faces = GetFaces();
-  for (int face_id = 0; face_id < faces.size(); face_id += 2) {
-    const auto& face = faces[face_id];
-    Vector3f center = get_face_center(face);
-    Vector3f normal = get_face_normal(face, center);
-    Vector3f w = gravity.cross(normal);
-    const float w_sq_norm = w.squaredNorm();
-    if (w_sq_norm < ground_plane_error) {
-      ground_plane_error = w_sq_norm;
-      ground_plane_id = face_id;
-    }
-  }
-
-  Vector3f center = get_face_center(faces[ground_plane_id]);
-  Vector3f normal = get_face_normal(faces[ground_plane_id], center);
-
-  // For each face, we also have a parallel face that it's normal is also
-  // aligned with gravity vector. We pick the face with lower height (y-value).
-  // The parallel to face 0 is 1, face 2 is 3, and face 4 is 5.
-  int parallel_face_id = ground_plane_id + 1;
-  const auto& parallel_face = faces[parallel_face_id];
-  Vector3f parallel_face_center = get_face_center(parallel_face);
-  Vector3f parallel_face_normal =
-      get_face_normal(parallel_face, parallel_face_center);
-  if (parallel_face_center[1] < center[1]) {
-    center = parallel_face_center;
-    normal = parallel_face_normal;
-  }
-  return {center, normal};
-}
-
-template <typename T>
-void Box::Fit(const std::vector<T>& vertices) {
-  CHECK_EQ(vertices.size(), kNumKeypoints);
-  scale_.setZero();
-  // The scale would remain invariant under rotation and translation.
-  // We can safely estimate the scale from the oriented box.
-  for (int axis = 0; axis < kNumberOfAxis; ++axis) {
-    for (int edge_id = 0; edge_id < kEdgesPerAxis; ++edge_id) {
-      // The edges are stored in quadruples according to each axis
-      const std::array<int, 2>& edge = edges_[axis * kEdgesPerAxis + edge_id];
-      scale_[axis] += (vertices[edge[0]] - vertices[edge[1]]).norm();
-    }
-    scale_[axis] /= kEdgesPerAxis;
-  }
-  // Create a scaled axis-aligned box
-  transformation_.setIdentity();
-  Update();
-
-  using MatrixN3_RM = Eigen::Matrix<float, kNumKeypoints, 3, Eigen::RowMajor>;
-  Eigen::Map<const MatrixN3_RM> v(vertices[0].data());
-  Eigen::Map<const MatrixN3_RM> system(bounding_box_[0].data());
-  auto system_h = system.rowwise().homogeneous().eval();
-  auto system_g = system_h.colPivHouseholderQr();
-  auto solution = system_g.solve(v).eval();
-  transformation_.topLeftCorner<3, 4>() = solution.transpose();
-  Update();
-}
-
-template void Box::Fit<Vector3f>(const std::vector<Vector3f>&);
-template void Box::Fit<Map<Vector3f>>(const std::vector<Map<Vector3f>>&);
-template void Box::Fit<Map<const Vector3f>>(
-    const std::vector<Map<const Vector3f>>&);
-}  // namespace mediapipe

mediapipe/graphs/object_detection_3d/calculators/box.h

@@ -1,132 +0,0 @@
-// Copyright 2020 The MediaPipe Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#ifndef MEDIAPIPE_GRAPHS_OBJECT_DETECTION_3D_BOX_H_
-#define MEDIAPIPE_GRAPHS_OBJECT_DETECTION_3D_BOX_H_
-
-#include <vector>
-
-#include "mediapipe/graphs/object_detection_3d/calculators/model.h"
-
-namespace mediapipe {
-
-// Model for the bounding box in 3D
-// The box has 9 degrees of freedom, which uniquely defines 8 keypoints in the
-// fixed world-coordinate system.
-//
-// The 8 keypoints are defined as follows
-//
-//  kp-id  axis
-//  0      000    ---
-//  1      001    --+
-//  2      010    -+-
-//  3      011    -++
-//  4      100    +--
-//  5      101    +-+
-//  6      110    ++-
-//  7      111    +++
-//
-// where xyz means positive or negative vector along the axis where the center
-// of the box is the origin. The resulting bounding box is
-//
-//              x                              x
-//      0 + + + + + + + + 4                 .-------
-//      +\                +\                |\
-//      + \ y             + \             z | \ y
-//      +  \              +  \              |  \
-//      +   2 + + + + + + + + 6
-//    z +   +             +   +
-//      +   +             +   +
-//      +   +     C       +   +
-//      +   +             +   +
-//      1 + + + + + + + + 5   +
-//       \  +              \  +
-//        \ +               \ +
-//         \+                \+
-//          3 + + + + + + + + 7
-//
-// World coordinate system: +y is up (aligned with gravity),
-// +z is toward the user, +x follows right hand rule.
-// The front face is defined as +z axis on xy plane.
-// The top face is defined as +y axis on xz plane.
-//
-
-class Box : public Model {
- public:
-  EIGEN_MAKE_ALIGNED_OPERATOR_NEW
-
-  explicit Box(const std::string& category);
-  ~Box() override = default;
-
-  bool InsideTest(const Vector3f& point, int check_axis) const;
-
-  const std::vector<Face>& GetFaces() const { return faces_; }
-  const Face& GetFace(size_t face_id) const { return faces_[face_id]; }
-
-  const std::vector<std::array<int, 2>>& GetEdges() const { return edges_; }
-  const std::array<int, 2>& GetEdge(size_t edge_id) const {
-    return edges_[edge_id];
-  }
-
-  // Returns the keypoints for the front face of the box.
-  // The front face is defind as a face with +z normal vector on xy plane
-  // In Box's c'tor, the top face is set to {1, 3, 7, 5}
-  const Face& GetFrontFace() const;
-
-  // Returns the keypoints for the top face of the box.
-  // The top face is defind as a face with +z normal vector on xy plane
-  // In Box's c'tor, the top face is set to {1, 3, 7, 5}
-  const Face& GetTopFace() const;
-
-  void Update() override;
-  void Adjust(const std::vector<float>& variables) override;
-  float* GetVertex(size_t vertex_id) override;
-  const float* GetVertex(size_t vertex_id) const override;
-  void Deserialize(const Object& obj) override;
-  void Serialize(Object* obj) override;
-
-  // Computes the plane center and the normal vector for the plane the object
-  // is sitting on in the world cooordinate system. The normal vector is roughly
-  // aligned with gravity.
-  std::pair<Vector3f, Vector3f> GetGroundPlane() const;
-
-  // Estimates a box 9-dof parameters from the given vertices. Directly computes
-  // the scale of the box, then solves for orientation and translation.
-  // Expects a std::vector of size 9 of a Eigen::Vector3f or mapped Vector3f.
-  // If mapping proto messages, we recommend to use the Map<const Vector3f>.
-  // For example:
-  //
-  // using T = Map<const Vector3f>;
-  // std::vector<T> vertices;
-  // for (const auto& point : message) { // point is a repeated float message.
-  //   T p(point.data());
-  //   vertices.emplace_back(p);
-  // }
-  // box.Fit<T>(vertices);
-  //
-  // The Points must be arranged as 1 + 8 (center keypoint followed by 8 box
-  // vertices) vector. This function will overwrite the scale and transformation
-  // properties of the class.
-  template <typename T = Eigen::Map<const Vector3f>>
-  void Fit(const std::vector<T>& vertices);
-
- private:
-  std::vector<Face> faces_;
-  std::vector<std::array<int, 2>> edges_;
-  std::vector<Vector3f> bounding_box_;
-};
-
-}  // namespace mediapipe
-
-#endif  // MEDIAPIPE_GRAPHS_OBJECT_DETECTION_3D_BOX_H_

mediapipe/graphs/object_detection_3d/calculators/box_util.cc

@@ -1,153 +0,0 @@
-// Copyright 2020 The MediaPipe Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "mediapipe/graphs/object_detection_3d/calculators/box_util.h"
-
-#include <math.h>
-
-#include "mediapipe/framework/port/logging.h"
-#include "mediapipe/framework/port/opencv_core_inc.h"
-#include "mediapipe/framework/port/opencv_imgproc_inc.h"
-#include "mediapipe/util/tracking/box_tracker.pb.h"
-
-namespace mediapipe {
-void ComputeBoundingRect(const std::vector<cv::Point2f>& points,
-                         mediapipe::TimedBoxProto* box) {
-  CHECK(box != nullptr);
-  float top = 1.0f;
-  float bottom = 0.0f;
-  float left = 1.0f;
-  float right = 0.0f;
-  for (const auto& point : points) {
-    top = std::min(top, point.y);
-    bottom = std::max(bottom, point.y);
-    left = std::min(left, point.x);
-    right = std::max(right, point.x);
-  }
-  box->set_top(top);
-  box->set_bottom(bottom);
-  box->set_left(left);
-  box->set_right(right);
-  // We are currently only doing axis aligned bounding box. If we need to
-  // compute rotated bounding box, then we need the original image aspect ratio,
-  // map back to original image space, compute cv::convexHull, then for each
-  // edge of the hull, rotate according to edge orientation, find the box.
-  box->set_rotation(0.0f);
-}
-
-float ComputeBoxIoU(const TimedBoxProto& box1, const TimedBoxProto& box2) {
-  cv::Point2f box1_center((box1.left() + box1.right()) * 0.5f,
-                          (box1.top() + box1.bottom()) * 0.5f);
-  cv::Size2f box1_size(box1.right() - box1.left(), box1.bottom() - box1.top());
-  cv::RotatedRect rect1(box1_center, box1_size,
-                        -box1.rotation() * 180.0f / M_PI);
-  cv::Point2f box2_center((box2.left() + box2.right()) * 0.5f,
-                          (box2.top() + box2.bottom()) * 0.5f);
-  cv::Size2f box2_size(box2.right() - box2.left(), box2.bottom() - box2.top());
-  cv::RotatedRect rect2(box2_center, box2_size,
-                        -box2.rotation() * 180.0f / M_PI);
-  std::vector<cv::Point2f> intersections_unsorted;
-  std::vector<cv::Point2f> intersections;
-  cv::rotatedRectangleIntersection(rect1, rect2, intersections_unsorted);
-  if (intersections_unsorted.size() < 3) {
-    return 0.0f;
-  }
-  cv::convexHull(intersections_unsorted, intersections);
-
-  // We use Shoelace formula to compute area of polygons.
-  float intersection_area = 0.0f;
-  for (int i = 0; i < intersections.size(); ++i) {
-    const auto& curr_pt = intersections[i];
-    const int i_next = (i + 1) == intersections.size() ? 0 : (i + 1);
-    const auto& next_pt = intersections[i_next];
-    intersection_area += (curr_pt.x * next_pt.y - next_pt.x * curr_pt.y);
-  }
-  intersection_area = std::abs(intersection_area) * 0.5f;
-
-  // Compute union area
-  const float union_area =
-      rect1.size.area() + rect2.size.area() - intersection_area + 1e-5f;
-
-  const float iou = intersection_area / union_area;
-  return iou;
-}
-
-std::vector<cv::Point2f> ComputeBoxCorners(const TimedBoxProto& box,
-                                           float width, float height) {
-  // Rotate 4 corner w.r.t. center.
-  const cv::Point2f center(0.5f * (box.left() + box.right()) * width,
-                           0.5f * (box.top() + box.bottom()) * height);
-  const std::vector<cv::Point2f> corners{
-      cv::Point2f(box.left() * width, box.top() * height),
-      cv::Point2f(box.left() * width, box.bottom() * height),
-      cv::Point2f(box.right() * width, box.bottom() * height),
-      cv::Point2f(box.right() * width, box.top() * height)};
-
-  const float cos_a = std::cos(box.rotation());
-  const float sin_a = std::sin(box.rotation());
-  std::vector<cv::Point2f> transformed_corners(4);
-  for (int k = 0; k < 4; ++k) {
-    // Scale and rotate w.r.t. center.
-    const cv::Point2f rad = corners[k] - center;
-    const cv::Point2f rot_rad(cos_a * rad.x - sin_a * rad.y,
-                              sin_a * rad.x + cos_a * rad.y);
-    transformed_corners[k] = center + rot_rad;
-    transformed_corners[k].x /= width;
-    transformed_corners[k].y /= height;
-  }
-  return transformed_corners;
-}
-
-cv::Mat PerspectiveTransformBetweenBoxes(const TimedBoxProto& src_box,
-                                         const TimedBoxProto& dst_box,
-                                         const float aspect_ratio) {
-  std::vector<cv::Point2f> box1_corners =
-      ComputeBoxCorners(src_box, /*width*/ aspect_ratio, /*height*/ 1.0f);
-  std::vector<cv::Point2f> box2_corners =
-      ComputeBoxCorners(dst_box, /*width*/ aspect_ratio, /*height*/ 1.0f);
-  cv::Mat affine_transform = cv::getPerspectiveTransform(
-      /*src*/ box1_corners, /*dst*/ box2_corners);
-  cv::Mat output_affine;
-  affine_transform.convertTo(output_affine, CV_32FC1);
-  return output_affine;
-}
-
-cv::Point2f MapPoint(const TimedBoxProto& src_box, const TimedBoxProto& dst_box,
-                     const cv::Point2f& src_point, float width, float height) {
-  const cv::Point2f src_center(
-      0.5f * (src_box.left() + src_box.right()) * width,
-      0.5f * (src_box.top() + src_box.bottom()) * height);
-  const cv::Point2f dst_center(
-      0.5f * (dst_box.left() + dst_box.right()) * width,
-      0.5f * (dst_box.top() + dst_box.bottom()) * height);
-  const float scale_x =
-      (dst_box.right() - dst_box.left()) / (src_box.right() - src_box.left());
-  const float scale_y =
-      (dst_box.bottom() - dst_box.top()) / (src_box.bottom() - src_box.top());
-  const float rotation = dst_box.rotation() - src_box.rotation();
-  const cv::Point2f rad =
-      cv::Point2f(src_point.x * width, src_point.y * height) - src_center;
-  const float rad_x = rad.x * scale_x;
-  const float rad_y = rad.y * scale_y;
-  const float cos_a = std::cos(rotation);
-  const float sin_a = std::sin(rotation);
-  const cv::Point2f rot_rad(cos_a * rad_x - sin_a * rad_y,
-                            sin_a * rad_x + cos_a * rad_y);
-  const cv::Point2f dst_point_image = dst_center + rot_rad;
-  const cv::Point2f dst_point(dst_point_image.x / width,
-                              dst_point_image.y / height);
-  return dst_point;
-}
-
-}  // namespace mediapipe

mediapipe/graphs/object_detection_3d/calculators/box_util.h

@@ -1,50 +0,0 @@
-// Copyright 2020 The MediaPipe Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#ifndef MEDIAPIPE_GRAPHS_OBJECT_DETECTION_3D_BOX_UTIL_H_
-#define MEDIAPIPE_GRAPHS_OBJECT_DETECTION_3D_BOX_UTIL_H_
-
-#include "mediapipe/framework/port/opencv_core_inc.h"
-#include "mediapipe/util/tracking/box_tracker.pb.h"
-
-namespace mediapipe {
-
-// This function fills the geometry of the TimedBoxProto. Id, timestamp etc.
-// need to be set outside this function.
-void ComputeBoundingRect(const std::vector<cv::Point2f>& points,
-                         mediapipe::TimedBoxProto* box);
-
-// This function computes the intersection over union between two boxes.
-float ComputeBoxIoU(const TimedBoxProto& box1, const TimedBoxProto& box2);
-
-// Computes corners of the box.
-// width and height are image width and height, which is typically
-// needed since the box is in normalized coordinates.
-std::vector<cv::Point2f> ComputeBoxCorners(const TimedBoxProto& box,
-                                           float width, float height);
-
-// Computes the perspective transform from box1 to box2.
-// The input argument aspect_ratio is width / height of the image.
-// The returned matrix should be a 3x3 matrix.
-cv::Mat PerspectiveTransformBetweenBoxes(const TimedBoxProto& src_box,
-                                         const TimedBoxProto& dst_box,
-                                         const float aspect_ratio);
-
-// Map point according to source and destination box location.
-cv::Point2f MapPoint(const TimedBoxProto& src_box, const TimedBoxProto& dst_box,
-                     const cv::Point2f& src_point, float width, float height);
-
-}  // namespace mediapipe
-
-#endif  // MEDIAPIPE_GRAPHS_OBJECT_DETECTION_3D_BOX_UTIL_H_

mediapipe/graphs/object_detection_3d/calculators/box_util_test.cc

@@ -1,123 +0,0 @@
-// Copyright 2020 The MediaPipe Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "mediapipe/graphs/object_detection_3d/calculators/box_util.h"
-
-#include "mediapipe/framework/port/gmock.h"
-#include "mediapipe/framework/port/gtest.h"
-#include "mediapipe/framework/port/opencv_core_inc.h"
-#include "mediapipe/util/tracking/box_tracker.pb.h"
-
-namespace mediapipe {
-namespace {
-
-TEST(BoxUtilTest, TestComputeBoundingRect) {
-  std::vector<cv::Point2f> points{
-      cv::Point2f(0.35f, 0.25f), cv::Point2f(0.3f, 0.3f),
-      cv::Point2f(0.2f, 0.4f),   cv::Point2f(0.3f, 0.1f),
-      cv::Point2f(0.2f, 0.2f),   cv::Point2f(0.5f, 0.3f),
-      cv::Point2f(0.4f, 0.4f),   cv::Point2f(0.5f, 0.1f),
-      cv::Point2f(0.4f, 0.2f)};
-  TimedBoxProto box;
-  ComputeBoundingRect(points, &box);
-  EXPECT_FLOAT_EQ(0.1f, box.top());
-  EXPECT_FLOAT_EQ(0.4f, box.bottom());
-  EXPECT_FLOAT_EQ(0.2f, box.left());
-  EXPECT_FLOAT_EQ(0.5f, box.right());
-}
-
-TEST(BoxUtilTest, TestComputeBoxIoU) {
-  TimedBoxProto box1;
-  box1.set_top(0.2f);
-  box1.set_bottom(0.6f);
-  box1.set_left(0.1f);
-  box1.set_right(0.3f);
-  box1.set_rotation(0.0f);
-  TimedBoxProto box2 = box1;
-  box2.set_rotation(/*pi/2*/ 1.570796f);
-  const float box_area =
-      (box1.bottom() - box1.top()) * (box1.right() - box1.left());
-  const float box_intersection =
-      (box1.right() - box1.left()) * (box1.right() - box1.left());
-  const float expected_iou =
-      box_intersection / (box_area * 2 - box_intersection);
-  EXPECT_NEAR(expected_iou, ComputeBoxIoU(box1, box2), 3e-5f);
-
-  TimedBoxProto box3;
-  box3.set_top(0.2f);
-  box3.set_bottom(0.6f);
-  box3.set_left(0.5f);
-  box3.set_right(0.7f);
-  EXPECT_NEAR(0.0f, ComputeBoxIoU(box1, box3), 3e-5f);
-}
-
-TEST(BoxUtilTest, TestPerspectiveTransformBetweenBoxes) {
-  TimedBoxProto box1;
-  const float height = 4.0f;
-  const float width = 3.0f;
-  box1.set_top(1.0f / height);
-  box1.set_bottom(2.0f / height);
-  box1.set_left(1.0f / width);
-  box1.set_right(2.0f / width);
-  TimedBoxProto box2;
-  box2.set_top(1.0f / height);
-  box2.set_bottom(2.0f / height);
-  box2.set_left(1.0f / width);
-  box2.set_right(2.0f / width);
-  box2.set_rotation(/*pi/4*/ -0.785398f);
-  cv::Mat transform =
-      PerspectiveTransformBetweenBoxes(box1, box2, width / height);
-  const float kTolerence = 1e-5f;
-  const cv::Vec3f original_position(1.5f / width, 1.0f / height, 1.0f);
-  const cv::Mat transformed_position = transform * cv::Mat(original_position);
-  EXPECT_NEAR(
-      (1.5f - 0.5f * std::sqrt(2) / 2.0f) / width,
-      transformed_position.at<float>(0) / transformed_position.at<float>(2),
-      kTolerence);
-  EXPECT_NEAR(
-      (1.5f - 0.5f * std::sqrt(2) / 2.0f) / height,
-      transformed_position.at<float>(1) / transformed_position.at<float>(2),
-      kTolerence);
-}
-
-TEST(BoxUtilTest, TestMapPoint) {
-  const float height = 4.0f;
-  const float width = 3.0f;
-  TimedBoxProto box1;
-  box1.set_top(1.0f / height);
-  box1.set_bottom(2.0f / height);
-  box1.set_left(1.0f / width);
-  box1.set_right(2.0f / width);
-  TimedBoxProto box2;
-  box2.set_top(1.0f / height);
-  box2.set_bottom(2.0f / height);
-  box2.set_left(1.0f / width);
-  box2.set_right(2.0f / width);
-  box2.set_rotation(/*pi/4*/ -0.785398f);
-
-  cv::Point2f src_point1(1.2f / width, 1.4f / height);
-  cv::Point2f src_point2(1.3f / width, 1.8f / height);
-  const float distance1 = std::sqrt(0.1 * 0.1 + 0.4 * 0.4);
-  cv::Point2f dst_point1 = MapPoint(box1, box2, src_point1, width, height);
-  cv::Point2f dst_point2 = MapPoint(box1, box2, src_point2, width, height);
-  const float distance2 =
-      std::sqrt((dst_point1.x * width - dst_point2.x * width) *
-                    (dst_point1.x * width - dst_point2.x * width) +
-                (dst_point1.y * height - dst_point2.y * height) *
-                    (dst_point1.y * height - dst_point2.y * height));
-  EXPECT_NEAR(distance1, distance2, 1e-5f);
-}
-
-}  // namespace
-}  // namespace mediapipe

mediapipe/graphs/object_detection_3d/calculators/camera_parameters.proto

@@ -1,47 +0,0 @@
-// Copyright 2020 The MediaPipe Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-syntax = "proto2";
-
-package mediapipe;
-
-message CameraParametersProto {
-  // This number is non-negative, it represents camera height above ground
-  // normalized by focal length.
-  optional float height_above_ground = 1 [default = 100.0];
-  // Width of image in portrait orientation normalized by focal length
-  optional float portrait_width = 2 [default = 1.0103];
-  // Height of image in portrait orientation normalized by focal length
-  optional float portrait_height = 3 [default = 1.3435];
-  enum ImageOrientation {
-    PORTRAIT_ORIENTATION = 0;
-    LANDSCAPE_ORIENTATION = 1;
-  }
-  // The input image orientation
-  optional ImageOrientation image_orientation = 4
-      [default = PORTRAIT_ORIENTATION];
-
-  // This defines the projection method from 2D screen to 3D.
-  enum ProjectionMode {
-    UNSPECIFIED = 0;
-    // Projects 2D point to ground plane (horizontal plane).
-    GROUND_PLANE = 1;
-    // Projects 2D point to sphere.
-    SPHERE = 2;
-  }
-  optional ProjectionMode projection_mode = 5 [default = GROUND_PLANE];
-  // Radius of sphere when using the SPHERE projection mode above.
-  // The value is normalized by focal length.
-  optional float projection_sphere_radius = 6 [default = 100.0];
-}

mediapipe/graphs/object_detection_3d/calculators/decoder.cc

@@ -1,257 +0,0 @@
-// Copyright 2020 The MediaPipe Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "mediapipe/graphs/object_detection_3d/calculators/decoder.h"
-
-#include <limits>
-
-#include "Eigen/Dense"
-#include "mediapipe/framework/port/canonical_errors.h"
-#include "mediapipe/framework/port/logging.h"
-#include "mediapipe/framework/port/opencv_imgproc_inc.h"
-#include "mediapipe/framework/port/status.h"
-#include "mediapipe/graphs/object_detection_3d/calculators/annotation_data.pb.h"
-
-namespace mediapipe {
-constexpr int Decoder::kNumOffsetmaps = 16;
-
-namespace {
-void SetPoint3d(float x, float y, float z, Point3D* point_3d) {
-  point_3d->set_x(x);
-  point_3d->set_y(y);
-  point_3d->set_z(z);
-}
-}  // namespace
-
-FrameAnnotation Decoder::DecodeBoundingBoxKeypoints(
-    const cv::Mat& heatmap, const cv::Mat& offsetmap) const {
-  CHECK_EQ(1, heatmap.channels());
-  CHECK_EQ(kNumOffsetmaps, offsetmap.channels());
-  CHECK_EQ(heatmap.cols, offsetmap.cols);
-  CHECK_EQ(heatmap.rows, offsetmap.rows);
-
-  const float offset_scale = std::min(offsetmap.cols, offsetmap.rows);
-  const std::vector<cv::Point> center_points = ExtractCenterKeypoints(heatmap);
-  std::vector<BeliefBox> boxes;
-  for (const auto& center_point : center_points) {
-    BeliefBox box;
-    box.box_2d.emplace_back(center_point.x, center_point.y);
-    const int center_x = static_cast<int>(std::round(center_point.x));
-    const int center_y = static_cast<int>(std::round(center_point.y));
-    box.belief = heatmap.at<float>(center_y, center_x);
-    if (config_.voting_radius() > 1) {
-      DecodeByVoting(heatmap, offsetmap, center_x, center_y, offset_scale,
-                     offset_scale, &box);
-    } else {
-      DecodeByPeak(offsetmap, center_x, center_y, offset_scale, offset_scale,
-                   &box);
-    }
-    if (IsNewBox(&boxes, &box)) {
-      boxes.push_back(std::move(box));
-    }
-  }
-
-  const float x_scale = 1.0f / offsetmap.cols;
-  const float y_scale = 1.0f / offsetmap.rows;
-  FrameAnnotation frame_annotations;
-  for (const auto& box : boxes) {
-    auto* object = frame_annotations.add_annotations();
-    for (const auto& point : box.box_2d) {
-      auto* point2d = object->add_keypoints()->mutable_point_2d();
-      point2d->set_x(point.first * x_scale);
-      point2d->set_y(point.second * y_scale);
-    }
-  }
-  return frame_annotations;
-}
-
-void Decoder::DecodeByPeak(const cv::Mat& offsetmap, int center_x, int center_y,
-                           float offset_scale_x, float offset_scale_y,
-                           BeliefBox* box) const {
-  const auto& offset = offsetmap.at<cv::Vec<float, kNumOffsetmaps>>(
-      /*row*/ center_y, /*col*/ center_x);
-  for (int i = 0; i < kNumOffsetmaps / 2; ++i) {
-    const float x_offset = offset[2 * i] * offset_scale_x;
-    const float y_offset = offset[2 * i + 1] * offset_scale_y;
-    box->box_2d.emplace_back(center_x + x_offset, center_y + y_offset);
-  }
-}
-
-void Decoder::DecodeByVoting(const cv::Mat& heatmap, const cv::Mat& offsetmap,
-                             int center_x, int center_y, float offset_scale_x,
-                             float offset_scale_y, BeliefBox* box) const {
-  // Votes at the center.
-  const auto& center_offset = offsetmap.at<cv::Vec<float, kNumOffsetmaps>>(
-      /*row*/ center_y, /*col*/ center_x);
-  std::vector<float> center_votes(kNumOffsetmaps, 0.f);
-  for (int i = 0; i < kNumOffsetmaps / 2; ++i) {
-    center_votes[2 * i] = center_x + center_offset[2 * i] * offset_scale_x;
-    center_votes[2 * i + 1] =
-        center_y + center_offset[2 * i + 1] * offset_scale_y;
-  }
-
-  // Find voting window.
-  int x_min = std::max(0, center_x - config_.voting_radius());
-  int y_min = std::max(0, center_y - config_.voting_radius());
-  int width = std::min(heatmap.cols - x_min, config_.voting_radius() * 2 + 1);
-  int height = std::min(heatmap.rows - y_min, config_.voting_radius() * 2 + 1);
-  cv::Rect rect(x_min, y_min, width, height);
-  cv::Mat heat = heatmap(rect);
-  cv::Mat offset = offsetmap(rect);
-
-  for (int i = 0; i < kNumOffsetmaps / 2; ++i) {
-    float x_sum = 0.f;
-    float y_sum = 0.f;
-    float votes = 0.f;
-    for (int r = 0; r < heat.rows; ++r) {
-      for (int c = 0; c < heat.cols; ++c) {
-        const float belief = heat.at<float>(r, c);
-        if (belief < config_.voting_threshold()) {
-          continue;
-        }
-        float offset_x =
-            offset.at<cv::Vec<float, kNumOffsetmaps>>(r, c)[2 * i] *
-            offset_scale_x;
-        float offset_y =
-            offset.at<cv::Vec<float, kNumOffsetmaps>>(r, c)[2 * i + 1] *
-            offset_scale_y;
-        float vote_x = c + rect.x + offset_x;
-        float vote_y = r + rect.y + offset_y;
-        float x_diff = std::abs(vote_x - center_votes[2 * i]);
-        float y_diff = std::abs(vote_y - center_votes[2 * i + 1]);
-        if (x_diff > config_.voting_allowance() ||
-            y_diff > config_.voting_allowance()) {
-          continue;
-        }
-        x_sum += vote_x * belief;
-        y_sum += vote_y * belief;
-        votes += belief;
-      }
-    }
-    box->box_2d.emplace_back(x_sum / votes, y_sum / votes);
-  }
-}
-
-bool Decoder::IsNewBox(std::vector<BeliefBox>* boxes, BeliefBox* box) const {
-  for (auto& b : *boxes) {
-    if (IsIdentical(b, *box)) {
-      if (b.belief < box->belief) {
-        std::swap(b, *box);
-      }
-      return false;
-    }
-  }
-  return true;
-}
-
-bool Decoder::IsIdentical(const BeliefBox& box_1,
-                          const BeliefBox& box_2) const {
-  // Skip the center point.
-  for (int i = 1; i < box_1.box_2d.size(); ++i) {
-    const float x_diff =
-        std::abs(box_1.box_2d[i].first - box_2.box_2d[i].first);
-    const float y_diff =
-        std::abs(box_1.box_2d[i].second - box_2.box_2d[i].second);
-    if (x_diff > config_.voting_allowance() ||
-        y_diff > config_.voting_allowance()) {
-      return false;
-    }
-  }
-  return true;
-}
-
-std::vector<cv::Point> Decoder::ExtractCenterKeypoints(
-    const cv::Mat& center_heatmap) const {
-  cv::Mat max_filtered_heatmap(center_heatmap.rows, center_heatmap.cols,
-                               center_heatmap.type());
-  const int kernel_size =
-      static_cast<int>(config_.local_max_distance() * 2 + 1 + 0.5f);
-  const cv::Size morph_size(kernel_size, kernel_size);
-  cv::dilate(center_heatmap, max_filtered_heatmap,
-             cv::getStructuringElement(cv::MORPH_RECT, morph_size));
-  cv::Mat peak_map;
-  cv::bitwise_and((center_heatmap >= max_filtered_heatmap),
-                  (center_heatmap >= config_.heatmap_threshold()), peak_map);
-  std::vector<cv::Point> locations;  // output, locations of non-zero pixels
-  cv::findNonZero(peak_map, locations);
-  return locations;
-}
-
-absl::Status Decoder::Lift2DTo3D(
-    const Eigen::Matrix<float, 4, 4, Eigen::RowMajor>& projection_matrix,
-    bool portrait, FrameAnnotation* estimated_box) const {
-  CHECK(estimated_box != nullptr);
-  const float fx = projection_matrix(0, 0);
-  const float fy = projection_matrix(1, 1);
-  const float cx = projection_matrix(0, 2);
-  const float cy = projection_matrix(1, 2);
-  for (auto& annotation : *estimated_box->mutable_annotations()) {
-    Eigen::Matrix<float, 16, 12, Eigen::RowMajor> m =
-        Eigen::Matrix<float, 16, 12, Eigen::RowMajor>::Zero(16, 12);
-    CHECK_EQ(9, annotation.keypoints_size());
-    float u, v;
-    for (int i = 0; i < 8; ++i) {
-      const auto& keypoint2d = annotation.keypoints(i + 1).point_2d();
-      if (portrait) {
-        // swap x and y given that our image is in portrait orientation
-        u = keypoint2d.y() * 2 - 1;
-        v = keypoint2d.x() * 2 - 1;
-      } else {
-        u = keypoint2d.x() * 2 - 1;
-        v = 1 - keypoint2d.y() * 2;  // (1 - keypoint2d.y()) * 2 - 1
-      }
-      for (int j = 0; j < 4; ++j) {
-        // For each of the 4 control points, formulate two rows of the
-        // m matrix (two equations).
-        const float control_alpha = epnp_alpha_(i, j);
-        m(i * 2, j * 3) = fx * control_alpha;
-        m(i * 2, j * 3 + 2) = (cx + u) * control_alpha;
-        m(i * 2 + 1, j * 3 + 1) = fy * control_alpha;
-        m(i * 2 + 1, j * 3 + 2) = (cy + v) * control_alpha;
-      }
-    }
-    // This is a self adjoint matrix. Use SelfAdjointEigenSolver for a fast
-    // and stable solution.
-    Eigen::Matrix<float, 12, 12, Eigen::RowMajor> mt_m = m.transpose() * m;
-    Eigen::SelfAdjointEigenSolver<Eigen::Matrix<float, 12, 12, Eigen::RowMajor>>
-        eigen_solver(mt_m);
-    if (eigen_solver.info() != Eigen::Success) {
-      return absl::AbortedError("Eigen decomposition failed.");
-    }
-    CHECK_EQ(12, eigen_solver.eigenvalues().size());
-    // Eigenvalues are sorted in increasing order for SelfAdjointEigenSolver
-    // only! If you use other Eigen Solvers, it's not guaranteed to be in
-    // increasing order. Here, we just take the eigen vector corresponding
-    // to first/smallest eigen value, since we used SelfAdjointEigenSolver.
-    Eigen::VectorXf eigen_vec = eigen_solver.eigenvectors().col(0);
-    Eigen::Map<Eigen::Matrix<float, 4, 3, Eigen::RowMajor>> control_matrix(
-        eigen_vec.data());
-    if (control_matrix(0, 2) > 0) {
-      control_matrix = -control_matrix;
-    }
-    // First set the center keypoint.
-    SetPoint3d(control_matrix(0, 0), control_matrix(0, 1), control_matrix(0, 2),
-               annotation.mutable_keypoints(0)->mutable_point_3d());
-    // Then set the 8 vertices.
-    Eigen::Matrix<float, 8, 3, Eigen::RowMajor> vertices =
-        epnp_alpha_ * control_matrix;
-    for (int i = 0; i < 8; ++i) {
-      SetPoint3d(vertices(i, 0), vertices(i, 1), vertices(i, 2),
-                 annotation.mutable_keypoints(i + 1)->mutable_point_3d());
-    }
-  }
-  return absl::OkStatus();
-}
-
-}  // namespace mediapipe

mediapipe/graphs/object_detection_3d/calculators/decoder.h

@@ -1,109 +0,0 @@
-// Copyright 2020 The MediaPipe Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#ifndef MEDIAPIPE_GRAPHS_OBJECT_DETECTION_3D_DECODER_H_
-#define MEDIAPIPE_GRAPHS_OBJECT_DETECTION_3D_DECODER_H_
-
-#include <vector>
-
-#include "Eigen/Dense"
-#include "absl/status/status.h"
-#include "mediapipe/framework/port/opencv_core_inc.h"
-#include "mediapipe/graphs/object_detection_3d/calculators/annotation_data.pb.h"
-#include "mediapipe/graphs/object_detection_3d/calculators/belief_decoder_config.pb.h"
-
-namespace mediapipe {
-
-// Decodes 3D bounding box from heatmaps and offset maps. In the future,
-// if we want to develop decoder for generic skeleton, then we need to
-// generalize this class, and make a few child classes.
-class Decoder {
- public:
-  static const int kNumOffsetmaps;
-
-  explicit Decoder(const BeliefDecoderConfig& config) : config_(config) {
-    epnp_alpha_ << 4.0f, -1.0f, -1.0f, -1.0f, 2.0f, -1.0f, -1.0f, 1.0f, 2.0f,
-        -1.0f, 1.0f, -1.0f, 0.0f, -1.0f, 1.0f, 1.0f, 2.0f, 1.0f, -1.0f, -1.0f,
-        0.0f, 1.0f, -1.0f, 1.0f, 0.0f, 1.0f, 1.0f, -1.0f, -2.0f, 1.0f, 1.0f,
-        1.0f;
-  }
-
-  // Decodes bounding boxes from predicted heatmap and offset maps.
-  // Input:
-  //   heatmap: a single channel cv::Mat representing center point heatmap
-  //   offsetmap: a 16 channel cv::Mat representing the 16 offset maps
-  //              (2 for each of the 8 vertices)
-  // Output:
-  //   Outputs 3D bounding boxes 2D vertices, represented by 'point_2d' field
-  //   in each 'keypoints' field of object annotations.
-  FrameAnnotation DecodeBoundingBoxKeypoints(const cv::Mat& heatmap,
-                                             const cv::Mat& offsetmap) const;
-
-  // Lifts the estimated 2D projections of bounding box vertices to 3D.
-  // This function uses the EPnP approach described in this paper:
-  // https://icwww.epfl.ch/~lepetit/papers/lepetit_ijcv08.pdf .
-  // Input:
-  //   projection_matrix: the projection matrix from 3D coordinate
-  //     to screen coordinate.
-  //     The 2D screen coordinate is defined as: u is along the long
-  //     edge of the device, pointing down; v is along the short edge
-  //     of the device, pointing right.
-  //   portrait: a boolen variable indicating whether our images are
-  //     obtained in portrait orientation or not.
-  //   estimated_box: annotation with point_2d field populated with
-  //     2d vertices.
-  // Output:
-  //   estimated_box: annotation with point_3d field populated with
-  //     3d vertices.
-  absl::Status Lift2DTo3D(
-      const Eigen::Matrix<float, 4, 4, Eigen::RowMajor>& projection_matrix,
-      bool portrait, FrameAnnotation* estimated_box) const;
-
- private:
-  struct BeliefBox {
-    float belief;
-    std::vector<std::pair<float, float>> box_2d;
-  };
-
-  std::vector<cv::Point> ExtractCenterKeypoints(
-      const cv::Mat& center_heatmap) const;
-
-  // Decodes 2D keypoints at the peak point.
-  void DecodeByPeak(const cv::Mat& offsetmap, int center_x, int center_y,
-                    float offset_scale_x, float offset_scale_y,
-                    BeliefBox* box) const;
-
-  // Decodes 2D keypoints by voting around the peak.
-  void DecodeByVoting(const cv::Mat& heatmap, const cv::Mat& offsetmap,
-                      int center_x, int center_y, float offset_scale_x,
-                      float offset_scale_y, BeliefBox* box) const;
-
-  // Returns true if it is a new box. Otherwise, it may replace an existing box
-  // if the new box's belief is higher.
-  bool IsNewBox(std::vector<BeliefBox>* boxes, BeliefBox* box) const;
-
-  // Returns true if the two boxes are identical.
-  bool IsIdentical(const BeliefBox& box_1, const BeliefBox& box_2) const;
-
-  BeliefDecoderConfig config_;
-  // Following equation (1) in this paper
-  // https://icwww.epfl.ch/~lepetit/papers/lepetit_ijcv08.pdf,
-  // this variable denotes the coefficients for the 4 control points
-  // for each of the 8 3D box vertices.
-  Eigen::Matrix<float, 8, 4, Eigen::RowMajor> epnp_alpha_;
-};
-
-}  // namespace mediapipe
-
-#endif  // MEDIAPIPE_GRAPHS_OBJECT_DETECTION_3D_DECODER_H_

mediapipe/graphs/object_detection_3d/calculators/filter_detection_calculator.cc

@@ -1,265 +0,0 @@
-// Copyright 2020 The MediaPipe Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include <memory>
-#include <string>
-#include <vector>
-
-#include "absl/container/node_hash_set.h"
-#include "absl/strings/str_split.h"
-#include "absl/strings/string_view.h"
-#include "absl/strings/strip.h"
-#include "mediapipe/framework/calculator_framework.h"
-#include "mediapipe/framework/formats/detection.pb.h"
-#include "mediapipe/framework/formats/location_data.pb.h"
-#include "mediapipe/framework/port/logging.h"
-#include "mediapipe/framework/port/map_util.h"
-#include "mediapipe/framework/port/re2.h"
-#include "mediapipe/framework/port/status.h"
-#include "mediapipe/graphs/object_detection_3d/calculators/filter_detection_calculator.pb.h"
-
-namespace mediapipe {
-
-namespace {
-
-constexpr char kDetectionTag[] = "DETECTION";
-constexpr char kDetectionsTag[] = "DETECTIONS";
-constexpr char kLabelsTag[] = "LABELS";
-constexpr char kLabelsCsvTag[] = "LABELS_CSV";
-
-using ::mediapipe::ContainsKey;
-using ::mediapipe::RE2;
-using Detections = std::vector<Detection>;
-using Strings = std::vector<std::string>;
-
-}  // namespace
-
-// Filters the entries in a Detection to only those with valid scores
-// for the specified allowed labels. Allowed labels are provided as a
-// vector<std::string> in an optional input side packet. Allowed labels can
-// contain simple strings or regular expressions. The valid score range
-// can be set in the options.The allowed labels can be provided as
-// vector<std::string> (LABELS) or CSV std::string (LABELS_CSV) containing class
-// names of allowed labels. Note: Providing an empty vector in the input side
-// packet Packet causes this calculator to act as a sink if
-// empty_allowed_labels_means_allow_everything is set to false (default value).
-// To allow all labels, use the calculator with no input side packet stream, or
-// set empty_allowed_labels_means_allow_everything to true.
-//
-// Example config:
-// node {
-//   calculator: "FilterDetectionCalculator"
-//   input_stream: "DETECTIONS:detections"
-//   output_stream: "DETECTIONS:filtered_detections"
-//   input_side_packet: "LABELS:allowed_labels"
-//   node_options: {
-//     [type.googleapis.com/mediapipe.FilterDetectionCalculatorOptions]: {
-//       min_score: 0.5
-//     }
-//   }
-// }
-
-struct FirstGreaterComparator {
-  bool operator()(const std::pair<float, int>& a,
-                  const std::pair<float, int>& b) const {
-    return a.first > b.first;
-  }
-};
-
-::mediapipe::Status SortLabelsByDecreasingScore(const Detection& detection,
-                                                Detection* sorted_detection) {
-  RET_CHECK(sorted_detection);
-  RET_CHECK_EQ(detection.score_size(), detection.label_size());
-  if (!detection.label_id().empty()) {
-    RET_CHECK_EQ(detection.score_size(), detection.label_id_size());
-  }
-  // Copies input to keep all fields unchanged, and to reserve space for
-  // repeated fields. Repeated fields (score, label, and label_id) will be
-  // overwritten.
-  *sorted_detection = detection;
-
-  std::vector<std::pair<float, int>> scores_and_indices(detection.score_size());
-  for (int i = 0; i < detection.score_size(); ++i) {
-    scores_and_indices[i].first = detection.score(i);
-    scores_and_indices[i].second = i;
-  }
-
-  std::sort(scores_and_indices.begin(), scores_and_indices.end(),
-            FirstGreaterComparator());
-
-  for (int i = 0; i < detection.score_size(); ++i) {
-    const int index = scores_and_indices[i].second;
-    sorted_detection->set_score(i, detection.score(index));
-    sorted_detection->set_label(i, detection.label(index));
-  }
-
-  if (!detection.label_id().empty()) {
-    for (int i = 0; i < detection.score_size(); ++i) {
-      const int index = scores_and_indices[i].second;
-      sorted_detection->set_label_id(i, detection.label_id(index));
-    }
-  }
-  return ::mediapipe::OkStatus();
-}
-
-class FilterDetectionCalculator : public CalculatorBase {
- public:
-  static ::mediapipe::Status GetContract(CalculatorContract* cc);
-  ::mediapipe::Status Open(CalculatorContext* cc) override;
-  ::mediapipe::Status Process(CalculatorContext* cc) override;
-
- private:
-  bool IsValidLabel(const std::string& label);
-  bool IsValidScore(float score);
-  // Stores numeric limits for filtering on the score.
-  FilterDetectionCalculatorOptions options_;
-  // We use the next two fields to possibly filter to a limited set of
-  // classes.  The hash_set will be empty in two cases: 1) if no input
-  // side packet stream is provided (not filtering on labels), or 2)
-  // if the input side packet contains an empty vector (no labels are
-  // allowed). We use limit_labels_ to distinguish between the two cases.
-  bool limit_labels_ = true;
-  absl::node_hash_set<std::string> allowed_labels_;
-};
-REGISTER_CALCULATOR(FilterDetectionCalculator);
-
-::mediapipe::Status FilterDetectionCalculator::GetContract(
-    CalculatorContract* cc) {
-  RET_CHECK(!cc->Inputs().GetTags().empty());
-  RET_CHECK(!cc->Outputs().GetTags().empty());
-
-  if (cc->Inputs().HasTag(kDetectionTag)) {
-    cc->Inputs().Tag(kDetectionTag).Set<Detection>();
-    cc->Outputs().Tag(kDetectionTag).Set<Detection>();
-  }
-  if (cc->Inputs().HasTag(kDetectionsTag)) {
-    cc->Inputs().Tag(kDetectionsTag).Set<Detections>();
-    cc->Outputs().Tag(kDetectionsTag).Set<Detections>();
-  }
-  if (cc->InputSidePackets().HasTag(kLabelsTag)) {
-    cc->InputSidePackets().Tag(kLabelsTag).Set<Strings>();
-  }
-  if (cc->InputSidePackets().HasTag(kLabelsCsvTag)) {
-    cc->InputSidePackets().Tag(kLabelsCsvTag).Set<std::string>();
-  }
-  return ::mediapipe::OkStatus();
-}
-
-::mediapipe::Status FilterDetectionCalculator::Open(CalculatorContext* cc) {
-  cc->SetOffset(TimestampDiff(0));
-  options_ = cc->Options<FilterDetectionCalculatorOptions>();
-  limit_labels_ = cc->InputSidePackets().HasTag(kLabelsTag) ||
-                  cc->InputSidePackets().HasTag(kLabelsCsvTag);
-  if (limit_labels_) {
-    Strings whitelist_labels;
-    if (cc->InputSidePackets().HasTag(kLabelsCsvTag)) {
-      whitelist_labels = absl::StrSplit(
-          cc->InputSidePackets().Tag(kLabelsCsvTag).Get<std::string>(), ',',
-          absl::SkipWhitespace());
-      for (auto& e : whitelist_labels) {
-        absl::StripAsciiWhitespace(&e);
-      }
-    } else {
-      whitelist_labels = cc->InputSidePackets().Tag(kLabelsTag).Get<Strings>();
-    }
-    allowed_labels_.insert(whitelist_labels.begin(), whitelist_labels.end());
-  }
-  if (limit_labels_ && allowed_labels_.empty()) {
-    if (options_.fail_on_empty_labels()) {
-      cc->GetCounter("VideosWithEmptyLabelsWhitelist")->Increment();
-      return tool::StatusFail(
-          "FilterDetectionCalculator received empty whitelist with "
-          "fail_on_empty_labels = true.");
-    }
-    if (options_.empty_allowed_labels_means_allow_everything()) {
-      // Continue as if side_input was not provided, i.e. pass all labels.
-      limit_labels_ = false;
-    }
-  }
-  return ::mediapipe::OkStatus();
-}
-
-::mediapipe::Status FilterDetectionCalculator::Process(CalculatorContext* cc) {
-  if (limit_labels_ && allowed_labels_.empty()) {
-    return ::mediapipe::OkStatus();
-  }
-  Detections detections;
-  if (cc->Inputs().HasTag(kDetectionsTag)) {
-    detections = cc->Inputs().Tag(kDetectionsTag).Get<Detections>();
-  } else if (cc->Inputs().HasTag(kDetectionTag)) {
-    detections.emplace_back(cc->Inputs().Tag(kDetectionsTag).Get<Detection>());
-  }
-  std::unique_ptr<Detections> outputs(new Detections);
-  for (const auto& input : detections) {
-    Detection output;
-    for (int i = 0; i < input.label_size(); ++i) {
-      const std::string& label = input.label(i);
-      const float score = input.score(i);
-      if (IsValidLabel(label) && IsValidScore(score)) {
-        output.add_label(label);
-        output.add_score(score);
-      }
-    }
-    if (output.label_size() > 0) {
-      if (input.has_location_data()) {
-        *output.mutable_location_data() = input.location_data();
-      }
-      Detection output_sorted;
-      if (!SortLabelsByDecreasingScore(output, &output_sorted).ok()) {
-        // Uses the orginal output if fails to sort.
-        cc->GetCounter("FailedToSortLabelsInDetection")->Increment();
-        output_sorted = output;
-      }
-      outputs->emplace_back(output_sorted);
-    }
-  }
-
-  if (cc->Outputs().HasTag(kDetectionsTag)) {
-    cc->Outputs()
-        .Tag(kDetectionsTag)
-        .Add(outputs.release(), cc->InputTimestamp());
-  } else if (!outputs->empty()) {
-    cc->Outputs()
-        .Tag(kDetectionsTag)
-        .Add(new Detection((*outputs)[0]), cc->InputTimestamp());
-  }
-  return ::mediapipe::OkStatus();
-}
-
-bool FilterDetectionCalculator::IsValidLabel(const std::string& label) {
-  bool match = !limit_labels_ || ContainsKey(allowed_labels_, label);
-  if (!match) {
-    // If no exact match is found, check for regular expression
-    // comparions in the allowed_labels.
-    for (const auto& label_regexp : allowed_labels_) {
-      match = match || RE2::FullMatch(label, RE2(label_regexp));
-    }
-  }
-  return match;
-}
-
-bool FilterDetectionCalculator::IsValidScore(float score) {
-  if (options_.has_min_score() && score < options_.min_score()) {
-    LOG(ERROR) << "Filter out detection with low score " << score;
-    return false;
-  }
-  if (options_.has_max_score() && score > options_.max_score()) {
-    LOG(ERROR) << "Filter out detection with high score " << score;
-    return false;
-  }
-  LOG(ERROR) << "Pass detection with score " << score;
-  return true;
-}
-
-}  // namespace mediapipe

mediapipe/graphs/object_detection_3d/calculators/filter_detection_calculator.proto

@@ -1,45 +0,0 @@
-// Copyright 2020 The MediaPipe Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-syntax = "proto2";
-
-package mediapipe;
-
-import "mediapipe/framework/calculator.proto";
-
-message FilterDetectionCalculatorOptions {
-  extend CalculatorOptions {
-    optional FilterDetectionCalculatorOptions ext = 339582987;
-  }
-  optional float min_score = 1;
-  optional float max_score = 2;
-  // Setting fail_on_empty_labels to true will cause the calculator to return a
-  // failure status on Open() if an empty list is provided on the external
-  // input, immediately terminating the graph run.
-  optional bool fail_on_empty_labels = 3 [default = false];
-  // If fail_on_empty_labels is set to false setting
-  // empty_allowed_labels_means_allow_everything to
-  // false will cause the calculator to close output stream and ignore remaining
-  // inputs if an empty list is provided. If
-  // empty_allowed_labels_means_allow_everything is set to true this will force
-  // calculator to pass all labels.
-  optional bool empty_allowed_labels_means_allow_everything = 6
-      [default = false];
-  // Determines whether the input format is a vector<Detection> (use-case object
-  // detectors) or Detection (use-case classifiers).
-  optional bool use_detection_vector = 4 [deprecated = true];
-  // Determines whether the input side packet format is a vector of labels, or
-  // a string with comma separated labels.
-  optional bool use_allowed_labels_csv = 5 [deprecated = true];
-}

mediapipe/graphs/object_detection_3d/calculators/frame_annotation_to_rect_calculator.cc

@@ -1,185 +0,0 @@
-// Copyright 2020 The MediaPipe Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-
-#include <cmath>
-
-#include "Eigen/Dense"
-#include "absl/memory/memory.h"
-#include "mediapipe/framework/calculator_framework.h"
-#include "mediapipe/framework/formats/rect.pb.h"
-#include "mediapipe/framework/port/ret_check.h"
-#include "mediapipe/framework/port/status.h"
-#include "mediapipe/graphs/object_detection_3d/calculators/annotation_data.pb.h"
-#include "mediapipe/graphs/object_detection_3d/calculators/box.h"
-#include "mediapipe/graphs/object_detection_3d/calculators/frame_annotation_to_rect_calculator.pb.h"
-
-namespace mediapipe {
-
-using Matrix3fRM = Eigen::Matrix<float, 3, 3, Eigen::RowMajor>;
-using Eigen::Vector2f;
-using Eigen::Vector3f;
-
-namespace {
-
-constexpr char kInputFrameAnnotationTag[] = "FRAME_ANNOTATION";
-constexpr char kOutputNormRectTag[] = "NORM_RECT";
-
-}  // namespace
-
-// A calculator that converts FrameAnnotation proto to NormalizedRect.
-// The rotation angle of the NormalizedRect is derived from object's 3d pose.
-// The angle is calculated such that after rotation the 2d projection of y-axis.
-// on the image plane is always vertical.
-class FrameAnnotationToRectCalculator : public CalculatorBase {
- public:
-  enum ViewStatus {
-    TOP_VIEW_ON,
-    TOP_VIEW_OFF,
-  };
-
-  static ::mediapipe::Status GetContract(CalculatorContract* cc);
-  ::mediapipe::Status Open(CalculatorContext* cc) override;
-  ::mediapipe::Status Process(CalculatorContext* cc) override;
-
- private:
-  void AnnotationToRect(const FrameAnnotation& annotation,
-                        NormalizedRect* rect);
-  float RotationAngleFromAnnotation(const FrameAnnotation& annotation);
-
-  float RotationAngleFromPose(const Matrix3fRM& rotation,
-                              const Vector3f& translation, const Vector3f& vec);
-  ViewStatus status_;
-  float off_threshold_;
-  float on_threshold_;
-};
-REGISTER_CALCULATOR(FrameAnnotationToRectCalculator);
-
-::mediapipe::Status FrameAnnotationToRectCalculator::Open(
-    CalculatorContext* cc) {
-  status_ = TOP_VIEW_OFF;
-  const auto& options = cc->Options<FrameAnnotationToRectCalculatorOptions>();
-  off_threshold_ = options.off_threshold();
-  on_threshold_ = options.on_threshold();
-  RET_CHECK(off_threshold_ <= on_threshold_);
-  return ::mediapipe::OkStatus();
-}
-
-::mediapipe::Status FrameAnnotationToRectCalculator::GetContract(
-    CalculatorContract* cc) {
-  RET_CHECK(!cc->Inputs().GetTags().empty());
-  RET_CHECK(!cc->Outputs().GetTags().empty());
-
-  if (cc->Inputs().HasTag(kInputFrameAnnotationTag)) {
-    cc->Inputs().Tag(kInputFrameAnnotationTag).Set<FrameAnnotation>();
-  }
-
-  if (cc->Outputs().HasTag(kOutputNormRectTag)) {
-    cc->Outputs().Tag(kOutputNormRectTag).Set<NormalizedRect>();
-  }
-  return ::mediapipe::OkStatus();
-}
-
-::mediapipe::Status FrameAnnotationToRectCalculator::Process(
-    CalculatorContext* cc) {
-  if (cc->Inputs().Tag(kInputFrameAnnotationTag).IsEmpty()) {
-    return ::mediapipe::OkStatus();
-  }
-  auto output_rect = absl::make_unique<NormalizedRect>();
-  AnnotationToRect(
-      cc->Inputs().Tag(kInputFrameAnnotationTag).Get<FrameAnnotation>(),
-      output_rect.get());
-
-  // Output
-  cc->Outputs()
-      .Tag(kOutputNormRectTag)
-      .Add(output_rect.release(), cc->InputTimestamp());
-  return ::mediapipe::OkStatus();
-}
-
-void FrameAnnotationToRectCalculator::AnnotationToRect(
-    const FrameAnnotation& annotation, NormalizedRect* rect) {
-  float x_min = std::numeric_limits<float>::max();
-  float x_max = std::numeric_limits<float>::min();
-  float y_min = std::numeric_limits<float>::max();
-  float y_max = std::numeric_limits<float>::min();
-  const auto& object = annotation.annotations(0);
-  for (const auto& keypoint : object.keypoints()) {
-    const auto& point_2d = keypoint.point_2d();
-    x_min = std::min(x_min, point_2d.x());
-    x_max = std::max(x_max, point_2d.x());
-    y_min = std::min(y_min, point_2d.y());
-    y_max = std::max(y_max, point_2d.y());
-  }
-  rect->set_x_center((x_min + x_max) / 2);
-  rect->set_y_center((y_min + y_max) / 2);
-  rect->set_width(x_max - x_min);
-  rect->set_height(y_max - y_min);
-  rect->set_rotation(RotationAngleFromAnnotation(annotation));
-}
-
-float FrameAnnotationToRectCalculator::RotationAngleFromAnnotation(
-    const FrameAnnotation& annotation) {
-  const auto& object = annotation.annotations(0);
-  Box box("category");
-  std::vector<Vector3f> vertices_3d;
-  std::vector<Vector2f> vertices_2d;
-  for (const auto& keypoint : object.keypoints()) {
-    const auto& point_3d = keypoint.point_3d();
-    const auto& point_2d = keypoint.point_2d();
-    vertices_3d.emplace_back(
-        Vector3f(point_3d.x(), point_3d.y(), point_3d.z()));
-    vertices_2d.emplace_back(Vector2f(point_2d.x(), point_2d.y()));
-  }
-  box.Fit(vertices_3d);
-  Vector3f scale = box.GetScale();
-  Matrix3fRM box_rotation = box.GetRotation();
-  Vector3f box_translation = box.GetTranslation();
-
-  // Rotation angle to use when top-view is on(top-view on),
-  // Which will make z-axis upright after the rotation.
-  const float angle_on =
-      RotationAngleFromPose(box_rotation, box_translation, Vector3f::UnitZ());
-  // Rotation angle to use when side-view is on(top-view off),
-  // Which will make y-axis upright after the rotation.
-  const float angle_off =
-      RotationAngleFromPose(box_rotation, box_translation, Vector3f::UnitY());
-
-  // Calculate angle between z-axis and viewing ray in degrees.
-  const float view_to_z_angle = std::acos(box_rotation(2, 1)) * 180 / M_PI;
-
-  // Determine threshold based on current status,
-  // on_threshold_ is used for TOP_VIEW_ON -> TOP_VIEW_OFF transition,
-  // off_threshold_ is used for TOP_VIEW_OFF -> TOP_VIEW_ON transition.
-  const float thresh =
-      (status_ == TOP_VIEW_ON) ? on_threshold_ : off_threshold_;
-
-  // If view_to_z_angle is smaller than threshold, then top-view is on;
-  // Otherwise top-view is off.
-  status_ = (view_to_z_angle < thresh) ? TOP_VIEW_ON : TOP_VIEW_OFF;
-
-  // Determine which angle to used based on current status_.
-  float angle_to_rotate = (status_ == TOP_VIEW_ON) ? angle_on : angle_off;
-  return angle_to_rotate;
-}
-
-float FrameAnnotationToRectCalculator::RotationAngleFromPose(
-    const Matrix3fRM& rotation, const Vector3f& translation,
-    const Vector3f& vec) {
-  auto p1 = rotation * vec + translation;
-  auto p2 = -rotation * vec + translation;
-  const float dy = p2[2] * p1[1] - p1[2] * p2[1];
-  const float dx = p2[2] * p1[0] - p1[2] * p2[0];
-  return std::atan2(-dy, dx);
-}
-
-}  // namespace mediapipe

mediapipe/graphs/object_detection_3d/calculators/frame_annotation_to_rect_calculator.proto

@@ -1,31 +0,0 @@
-// Copyright 2020 The MediaPipe Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-syntax = "proto2";
-
-package mediapipe;
-
-import "mediapipe/framework/calculator.proto";
-
-message FrameAnnotationToRectCalculatorOptions {
-  extend CalculatorOptions {
-    optional FrameAnnotationToRectCalculatorOptions ext = 338119067;
-  }
-
-  // The threshold to use when top-view is off,to enable hysteresis,
-  // It's required that off_threshold <= on_threshold.
-  optional float off_threshold = 1 [default = 40.0];
-  // The threshold to use when top-view is on.
-  optional float on_threshold = 2 [default = 41.0];
-}

mediapipe/graphs/object_detection_3d/calculators/frame_annotation_to_timed_box_list_calculator.cc

@@ -1,115 +0,0 @@
-// Copyright 2020 The MediaPipe Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include <utility>
-
-#include "absl/memory/memory.h"
-#include "mediapipe/framework/calculator_framework.h"
-#include "mediapipe/framework/port/opencv_core_inc.h"
-#include "mediapipe/framework/port/opencv_imgproc_inc.h"
-#include "mediapipe/framework/port/ret_check.h"
-#include "mediapipe/framework/port/status.h"
-#include "mediapipe/graphs/object_detection_3d/calculators/annotation_data.pb.h"
-#include "mediapipe/graphs/object_detection_3d/calculators/box_util.h"
-#include "mediapipe/util/tracking/box_tracker.pb.h"
-
-namespace {
-constexpr char kInputStreamTag[] = "FRAME_ANNOTATION";
-constexpr char kOutputStreamTag[] = "BOXES";
-}  // namespace
-
-namespace mediapipe {
-
-// Convert FrameAnnotation 3d bounding box detections to TimedBoxListProto
-// 2d bounding boxes.
-//
-// Input:
-//  FRAME_ANNOTATION - 3d bounding box annotation.
-// Output:
-//  BOXES - 2d bounding box enclosing the projection of 3d box.
-//
-// Usage example:
-// node {
-//   calculator: "FrameAnnotationToTimedBoxListCalculator"
-//   input_stream: "FRAME_ANNOTATION:frame_annotation"
-//   output_stream: "BOXES:boxes"
-// }
-class FrameAnnotationToTimedBoxListCalculator : public CalculatorBase {
- public:
-  static ::mediapipe::Status GetContract(CalculatorContract* cc);
-
-  ::mediapipe::Status Open(CalculatorContext* cc) override;
-  ::mediapipe::Status Process(CalculatorContext* cc) override;
-  ::mediapipe::Status Close(CalculatorContext* cc) override;
-};
-REGISTER_CALCULATOR(FrameAnnotationToTimedBoxListCalculator);
-
-::mediapipe::Status FrameAnnotationToTimedBoxListCalculator::GetContract(
-    CalculatorContract* cc) {
-  RET_CHECK(!cc->Inputs().GetTags().empty());
-  RET_CHECK(!cc->Outputs().GetTags().empty());
-
-  if (cc->Inputs().HasTag(kInputStreamTag)) {
-    cc->Inputs().Tag(kInputStreamTag).Set<FrameAnnotation>();
-  }
-
-  if (cc->Outputs().HasTag(kOutputStreamTag)) {
-    cc->Outputs().Tag(kOutputStreamTag).Set<TimedBoxProtoList>();
-  }
-  return ::mediapipe::OkStatus();
-}
-
-::mediapipe::Status FrameAnnotationToTimedBoxListCalculator::Open(
-    CalculatorContext* cc) {
-  return ::mediapipe::OkStatus();
-}
-
-::mediapipe::Status FrameAnnotationToTimedBoxListCalculator::Process(
-    CalculatorContext* cc) {
-  if (cc->Inputs().HasTag(kInputStreamTag) &&
-      !cc->Inputs().Tag(kInputStreamTag).IsEmpty()) {
-    const auto& frame_annotation =
-        cc->Inputs().Tag(kInputStreamTag).Get<FrameAnnotation>();
-    auto output_objects = absl::make_unique<TimedBoxProtoList>();
-    for (const auto& annotation : frame_annotation.annotations()) {
-      std::vector<cv::Point2f> key_points;
-      for (const auto& keypoint : annotation.keypoints()) {
-        key_points.push_back(
-            cv::Point2f(keypoint.point_2d().x(), keypoint.point_2d().y()));
-      }
-      TimedBoxProto* added_box = output_objects->add_box();
-      ComputeBoundingRect(key_points, added_box);
-      added_box->set_id(annotation.object_id());
-      const int64 time_msec =
-          static_cast<int64>(std::round(frame_annotation.timestamp() / 1000));
-      added_box->set_time_msec(time_msec);
-    }
-
-    // Output
-    if (cc->Outputs().HasTag(kOutputStreamTag)) {
-      cc->Outputs()
-          .Tag(kOutputStreamTag)
-          .Add(output_objects.release(), cc->InputTimestamp());
-    }
-  }
-
-  return ::mediapipe::OkStatus();
-}
-
-::mediapipe::Status FrameAnnotationToTimedBoxListCalculator::Close(
-    CalculatorContext* cc) {
-  return ::mediapipe::OkStatus();
-}
-
-}  // namespace mediapipe

mediapipe/graphs/object_detection_3d/calculators/frame_annotation_tracker.cc

@@ -1,102 +0,0 @@
-// Copyright 2020 The MediaPipe Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "mediapipe/graphs/object_detection_3d/calculators/frame_annotation_tracker.h"
-
-#include "absl/container/flat_hash_set.h"
-#include "mediapipe/framework/port/logging.h"
-#include "mediapipe/graphs/object_detection_3d/calculators/annotation_data.pb.h"
-#include "mediapipe/graphs/object_detection_3d/calculators/box_util.h"
-#include "mediapipe/util/tracking/box_tracker.pb.h"
-
-namespace mediapipe {
-
-void FrameAnnotationTracker::AddDetectionResult(
-    const FrameAnnotation& frame_annotation) {
-  const int64 time_us =
-      static_cast<int64>(std::round(frame_annotation.timestamp()));
-  for (const auto& object_annotation : frame_annotation.annotations()) {
-    detected_objects_[time_us + object_annotation.object_id()] =
-        object_annotation;
-  }
-}
-
-FrameAnnotation FrameAnnotationTracker::ConsolidateTrackingResult(
-    const TimedBoxProtoList& tracked_boxes,
-    absl::flat_hash_set<int>* cancel_object_ids) {
-  CHECK(cancel_object_ids != nullptr);
-  FrameAnnotation frame_annotation;
-  std::vector<int64> keys_to_be_deleted;
-  for (const auto& detected_obj : detected_objects_) {
-    const int object_id = detected_obj.second.object_id();
-    if (cancel_object_ids->contains(object_id)) {
-      // Remember duplicated detections' keys.
-      keys_to_be_deleted.push_back(detected_obj.first);
-      continue;
-    }
-    TimedBoxProto ref_box;
-    for (const auto& box : tracked_boxes.box()) {
-      if (box.id() == object_id) {
-        ref_box = box;
-        break;
-      }
-    }
-    if (!ref_box.has_id() || ref_box.id() < 0) {
-      LOG(ERROR) << "Can't find matching tracked box for object id: "
-                 << object_id << ". Likely lost tracking of it.";
-      keys_to_be_deleted.push_back(detected_obj.first);
-      continue;
-    }
-
-    // Find duplicated boxes
-    for (const auto& box : tracked_boxes.box()) {
-      if (box.id() != object_id) {
-        if (ComputeBoxIoU(ref_box, box) > iou_threshold_) {
-          cancel_object_ids->insert(box.id());
-        }
-      }
-    }
-
-    // Map ObjectAnnotation from detection to tracked time.
-    // First, gather all keypoints from source detection.
-    std::vector<cv::Point2f> key_points;
-    for (const auto& keypoint : detected_obj.second.keypoints()) {
-      key_points.push_back(
-          cv::Point2f(keypoint.point_2d().x(), keypoint.point_2d().y()));
-    }
-    // Second, find source box.
-    TimedBoxProto src_box;
-    ComputeBoundingRect(key_points, &src_box);
-    ObjectAnnotation* tracked_obj = frame_annotation.add_annotations();
-    tracked_obj->set_object_id(ref_box.id());
-    // Finally, map all keypoints in the source detection to tracked location.
-    for (const auto& keypoint : detected_obj.second.keypoints()) {
-      cv::Point2f dst = MapPoint(
-          src_box, ref_box,
-          cv::Point2f(keypoint.point_2d().x(), keypoint.point_2d().y()),
-          img_width_, img_height_);
-      auto* dst_point = tracked_obj->add_keypoints()->mutable_point_2d();
-      dst_point->set_x(dst.x);
-      dst_point->set_y(dst.y);
-    }
-  }
-
-  for (const auto& key : keys_to_be_deleted) {
-    detected_objects_.erase(key);
-  }
-
-  return frame_annotation;
-}
-
-}  // namespace mediapipe

mediapipe/graphs/object_detection_3d/calculators/frame_annotation_tracker.h

@@ -1,62 +0,0 @@
-// Copyright 2020 The MediaPipe Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#ifndef MEDIAPIPE_GRAPHS_OBJECT_DETECTION_3D_FRAME_ANNOTATION_TRACKER_H_
-#define MEDIAPIPE_GRAPHS_OBJECT_DETECTION_3D_FRAME_ANNOTATION_TRACKER_H_
-
-#include <functional>
-
-#include "absl/container/btree_map.h"
-#include "absl/container/flat_hash_set.h"
-#include "mediapipe/framework/port/integral_types.h"
-#include "mediapipe/graphs/object_detection_3d/calculators/annotation_data.pb.h"
-#include "mediapipe/util/tracking/box_tracker.pb.h"
-
-namespace mediapipe {
-
-class FrameAnnotationTracker {
- public:
-  // If two bounding boxes have IoU over iou_threshold, then we consider them
-  // describing the same object.
-  FrameAnnotationTracker(float iou_threshold, float img_width, float img_height)
-      : iou_threshold_(iou_threshold),
-        img_width_(img_width),
-        img_height_(img_height) {}
-
-  // Adds detection results from an external detector.
-  void AddDetectionResult(const FrameAnnotation& frame_annotation);
-
-  // Consolidates tracking result from an external tracker, associates with
-  // the detection result by the object id, and produces the corresponding
-  // result in FrameAnnotation. When there are duplicates, output the ids that
-  // need to be cancelled in cancel_object_ids.
-  // Note that the returned FrameAnnotation is missing timestamp. Need to fill
-  // that field.
-  FrameAnnotation ConsolidateTrackingResult(
-      const TimedBoxProtoList& tracked_boxes,
-      absl::flat_hash_set<int>* cancel_object_ids);
-
- private:
-  float iou_threshold_;
-  float img_width_;
-  float img_height_;
-  // Cached detection results over time.
-  // Key is timestamp_us + object_id.
-  absl::btree_map<int64, ObjectAnnotation, std::greater<int64>>
-      detected_objects_;
-};
-
-}  // namespace mediapipe
-
-#endif  // MEDIAPIPE_GRAPHS_OBJECT_DETECTION_3D_FRAME_ANNOTATION_TRACKER_H_

mediapipe/graphs/object_detection_3d/calculators/frame_annotation_tracker_calculator.cc

@@ -1,137 +0,0 @@
-// Copyright 2020 The MediaPipe Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "absl/container/flat_hash_set.h"
-#include "absl/memory/memory.h"
-#include "mediapipe/framework/calculator_framework.h"
-#include "mediapipe/framework/port/ret_check.h"
-#include "mediapipe/framework/port/status.h"
-#include "mediapipe/graphs/object_detection_3d/calculators/annotation_data.pb.h"
-#include "mediapipe/graphs/object_detection_3d/calculators/frame_annotation_tracker.h"
-#include "mediapipe/graphs/object_detection_3d/calculators/frame_annotation_tracker_calculator.pb.h"
-#include "mediapipe/util/tracking/box_tracker.pb.h"
-
-namespace {
-constexpr char kInputFrameAnnotationTag[] = "FRAME_ANNOTATION";
-constexpr char kInputTrackedBoxesTag[] = "TRACKED_BOXES";
-constexpr char kOutputTrackedFrameAnnotationTag[] = "TRACKED_FRAME_ANNOTATION";
-constexpr char kOutputCancelObjectIdTag[] = "CANCEL_OBJECT_ID";
-}  // namespace
-
-namespace mediapipe {
-
-// Tracks frame annotations seeded/updated by FRAME_ANNOTATION input_stream.
-// When using this calculator, make sure FRAME_ANNOTATION and TRACKED_BOXES
-// are in different sync set.
-//
-// Input:
-//  FRAME_ANNOTATION - frame annotation.
-//  TRACKED_BOXES - 2d box tracking result
-// Output:
-//  TRACKED_FRAME_ANNOTATION - annotation inferred from 2d tracking result.
-//  CANCEL_OBJECT_ID - object id that needs to be cancelled from the tracker.
-//
-// Usage example:
-// node {
-//   calculator: "FrameAnnotationTrackerCalculator"
-//   input_stream: "FRAME_ANNOTATION:frame_annotation"
-//   input_stream: "TRACKED_BOXES:tracked_boxes"
-//   output_stream: "TRACKED_FRAME_ANNOTATION:tracked_frame_annotation"
-//   output_stream: "CANCEL_OBJECT_ID:cancel_object_id"
-// }
-class FrameAnnotationTrackerCalculator : public CalculatorBase {
- public:
-  static ::mediapipe::Status GetContract(CalculatorContract* cc);
-
-  ::mediapipe::Status Open(CalculatorContext* cc) override;
-  ::mediapipe::Status Process(CalculatorContext* cc) override;
-  ::mediapipe::Status Close(CalculatorContext* cc) override;
-
- private:
-  std::unique_ptr<FrameAnnotationTracker> frame_annotation_tracker_;
-};
-REGISTER_CALCULATOR(FrameAnnotationTrackerCalculator);
-
-::mediapipe::Status FrameAnnotationTrackerCalculator::GetContract(
-    CalculatorContract* cc) {
-  RET_CHECK(!cc->Inputs().GetTags().empty());
-  RET_CHECK(!cc->Outputs().GetTags().empty());
-
-  if (cc->Inputs().HasTag(kInputFrameAnnotationTag)) {
-    cc->Inputs().Tag(kInputFrameAnnotationTag).Set<FrameAnnotation>();
-  }
-  if (cc->Inputs().HasTag(kInputTrackedBoxesTag)) {
-    cc->Inputs().Tag(kInputTrackedBoxesTag).Set<TimedBoxProtoList>();
-  }
-  if (cc->Outputs().HasTag(kOutputTrackedFrameAnnotationTag)) {
-    cc->Outputs().Tag(kOutputTrackedFrameAnnotationTag).Set<FrameAnnotation>();
-  }
-  if (cc->Outputs().HasTag(kOutputCancelObjectIdTag)) {
-    cc->Outputs().Tag(kOutputCancelObjectIdTag).Set<int>();
-  }
-  return ::mediapipe::OkStatus();
-}
-
-::mediapipe::Status FrameAnnotationTrackerCalculator::Open(
-    CalculatorContext* cc) {
-  const auto& options = cc->Options<FrameAnnotationTrackerCalculatorOptions>();
-  frame_annotation_tracker_ = absl::make_unique<FrameAnnotationTracker>(
-      options.iou_threshold(), options.img_width(), options.img_height());
-  return ::mediapipe::OkStatus();
-}
-
-::mediapipe::Status FrameAnnotationTrackerCalculator::Process(
-    CalculatorContext* cc) {
-  if (cc->Inputs().HasTag(kInputFrameAnnotationTag) &&
-      !cc->Inputs().Tag(kInputFrameAnnotationTag).IsEmpty()) {
-    frame_annotation_tracker_->AddDetectionResult(
-        cc->Inputs().Tag(kInputFrameAnnotationTag).Get<FrameAnnotation>());
-  }
-  if (cc->Inputs().HasTag(kInputTrackedBoxesTag) &&
-      !cc->Inputs().Tag(kInputTrackedBoxesTag).IsEmpty() &&
-      cc->Outputs().HasTag(kOutputTrackedFrameAnnotationTag)) {
-    absl::flat_hash_set<int> cancel_object_ids;
-    auto output_frame_annotation = absl::make_unique<FrameAnnotation>();
-    *output_frame_annotation =
-        frame_annotation_tracker_->ConsolidateTrackingResult(
-            cc->Inputs().Tag(kInputTrackedBoxesTag).Get<TimedBoxProtoList>(),
-            &cancel_object_ids);
-    output_frame_annotation->set_timestamp(cc->InputTimestamp().Microseconds());
-
-    cc->Outputs()
-        .Tag(kOutputTrackedFrameAnnotationTag)
-        .Add(output_frame_annotation.release(), cc->InputTimestamp());
-
-    if (cc->Outputs().HasTag(kOutputCancelObjectIdTag)) {
-      auto packet_timestamp = cc->InputTimestamp();
-      for (const auto& id : cancel_object_ids) {
-        // The timestamp is incremented (by 1 us) because currently the box
-        // tracker calculator only accepts one cancel object ID for any given
-        // timestamp.
-        cc->Outputs()
-            .Tag(kOutputCancelObjectIdTag)
-            .AddPacket(mediapipe::MakePacket<int>(id).At(packet_timestamp++));
-      }
-    }
-  }
-
-  return ::mediapipe::OkStatus();
-}
-
-::mediapipe::Status FrameAnnotationTrackerCalculator::Close(
-    CalculatorContext* cc) {
-  return ::mediapipe::OkStatus();
-}
-
-}  // namespace mediapipe

mediapipe/graphs/object_detection_3d/calculators/frame_annotation_tracker_calculator.proto

@@ -1,36 +0,0 @@
-// Copyright 2020 The MediaPipe Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// The option proto for the FrameAnnotationTrackerCalculatorOptions.
-
-syntax = "proto2";
-
-package mediapipe;
-
-import "mediapipe/framework/calculator.proto";
-
-message FrameAnnotationTrackerCalculatorOptions {
-  extend CalculatorOptions {
-    optional FrameAnnotationTrackerCalculatorOptions ext = 291291253;
-  }
-
-  // The threshold on intersection-over-union (IoU). We consider
-  // boxes with IoU larger than this threshold to be the duplicates.
-  optional float iou_threshold = 1 [default = 0.5];
-
-  // We need image dimension to properly compute annotation locations.
-  optional float img_width = 2;
-
-  optional float img_height = 3;
-}

mediapipe/graphs/object_detection_3d/calculators/frame_annotation_tracker_test.cc

@@ -1,143 +0,0 @@
-// Copyright 2020 The MediaPipe Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "mediapipe/graphs/object_detection_3d/calculators/frame_annotation_tracker.h"
-
-#include "absl/container/flat_hash_set.h"
-#include "mediapipe/framework/port/gmock.h"
-#include "mediapipe/framework/port/gtest.h"
-#include "mediapipe/framework/port/logging.h"
-#include "mediapipe/graphs/object_detection_3d/calculators/annotation_data.pb.h"
-#include "mediapipe/util/tracking/box_tracker.pb.h"
-
-namespace mediapipe {
-namespace {
-
-// Create a new object annotation by shifting a reference
-// object annotation.
-ObjectAnnotation ShiftObject2d(const ObjectAnnotation& ref_obj, float dx,
-                               float dy) {
-  ObjectAnnotation obj = ref_obj;
-  for (auto& keypoint : *(obj.mutable_keypoints())) {
-    const float ref_x = keypoint.point_2d().x();
-    const float ref_y = keypoint.point_2d().y();
-    keypoint.mutable_point_2d()->set_x(ref_x + dx);
-    keypoint.mutable_point_2d()->set_y(ref_y + dy);
-  }
-  return obj;
-}
-
-TimedBoxProto ShiftBox(const TimedBoxProto& ref_box, float dx, float dy) {
-  TimedBoxProto box = ref_box;
-  box.set_top(ref_box.top() + dy);
-  box.set_bottom(ref_box.bottom() + dy);
-  box.set_left(ref_box.left() + dx);
-  box.set_right(ref_box.right() + dx);
-  return box;
-}
-
-// Constructs a fixed ObjectAnnotation.
-ObjectAnnotation ConstructFixedObject(
-    const std::vector<std::vector<float>>& points) {
-  ObjectAnnotation obj;
-  for (const auto& point : points) {
-    auto* keypoint = obj.add_keypoints();
-    CHECK_EQ(2, point.size());
-    keypoint->mutable_point_2d()->set_x(point[0]);
-    keypoint->mutable_point_2d()->set_y(point[1]);
-  }
-  return obj;
-}
-
-TEST(FrameAnnotationTrackerTest, TestConsolidation) {
-  // Add 4 detections represented by FrameAnnotation, of which 3 correspond
-  // to the same object.
-  ObjectAnnotation object1, object2, object3, object4;
-  // The bounding rectangle for these object keypoints is:
-  // x: [0.2, 0.5], y: [0.1, 0.4]
-  object3 = ConstructFixedObject({{0.35f, 0.25f},
-                                  {0.3f, 0.3f},
-                                  {0.2f, 0.4f},
-                                  {0.3f, 0.1f},
-                                  {0.2f, 0.2f},
-                                  {0.5f, 0.3f},
-                                  {0.4f, 0.4f},
-                                  {0.5f, 0.1f},
-                                  {0.4f, 0.2f}});
-  object3.set_object_id(3);
-  object1 = ShiftObject2d(object3, -0.05f, -0.05f);
-  object1.set_object_id(1);
-  object2 = ShiftObject2d(object3, 0.05f, 0.05f);
-  object2.set_object_id(2);
-  object4 = ShiftObject2d(object3, 0.2f, 0.2f);
-  object4.set_object_id(4);
-  FrameAnnotation frame_annotation_1;
-  frame_annotation_1.set_timestamp(30 * 1000);  // 30ms
-  *(frame_annotation_1.add_annotations()) = object1;
-  *(frame_annotation_1.add_annotations()) = object4;
-  FrameAnnotation frame_annotation_2;
-  frame_annotation_2.set_timestamp(60 * 1000);  // 60ms
-  *(frame_annotation_2.add_annotations()) = object2;
-  FrameAnnotation frame_annotation_3;
-  frame_annotation_3.set_timestamp(90 * 1000);  // 90ms
-  *(frame_annotation_3.add_annotations()) = object3;
-
-  FrameAnnotationTracker frame_annotation_tracker(/*iou_threshold*/ 0.5f, 1.0f,
-                                                  1.0f);
-  frame_annotation_tracker.AddDetectionResult(frame_annotation_1);
-  frame_annotation_tracker.AddDetectionResult(frame_annotation_2);
-  frame_annotation_tracker.AddDetectionResult(frame_annotation_3);
-
-  TimedBoxProtoList timed_box_proto_list;
-  TimedBoxProto* timed_box_proto = timed_box_proto_list.add_box();
-  timed_box_proto->set_top(0.4f);
-  timed_box_proto->set_bottom(0.7f);
-  timed_box_proto->set_left(0.6f);
-  timed_box_proto->set_right(0.9f);
-  timed_box_proto->set_id(3);
-  timed_box_proto->set_time_msec(150);
-  timed_box_proto = timed_box_proto_list.add_box();
-  *timed_box_proto = ShiftBox(timed_box_proto_list.box(0), 0.01f, 0.01f);
-  timed_box_proto->set_id(1);
-  timed_box_proto->set_time_msec(150);
-  timed_box_proto = timed_box_proto_list.add_box();
-  *timed_box_proto = ShiftBox(timed_box_proto_list.box(0), -0.01f, -0.01f);
-  timed_box_proto->set_id(2);
-  timed_box_proto->set_time_msec(150);
-  absl::flat_hash_set<int> cancel_object_ids;
-  FrameAnnotation tracked_detection =
-      frame_annotation_tracker.ConsolidateTrackingResult(timed_box_proto_list,
-                                                         &cancel_object_ids);
-  EXPECT_EQ(2, cancel_object_ids.size());
-  EXPECT_EQ(1, cancel_object_ids.count(1));
-  EXPECT_EQ(1, cancel_object_ids.count(2));
-  EXPECT_EQ(1, tracked_detection.annotations_size());
-  EXPECT_EQ(3, tracked_detection.annotations(0).object_id());
-  EXPECT_EQ(object3.keypoints_size(),
-            tracked_detection.annotations(0).keypoints_size());
-  const float x_offset = 0.4f;
-  const float y_offset = 0.3f;
-  const float tolerance = 1e-5f;
-  for (int i = 0; i < object3.keypoints_size(); ++i) {
-    const auto& point_2d =
-        tracked_detection.annotations(0).keypoints(i).point_2d();
-    EXPECT_NEAR(point_2d.x(), object3.keypoints(i).point_2d().x() + x_offset,
-                tolerance);
-    EXPECT_NEAR(point_2d.y(), object3.keypoints(i).point_2d().y() + y_offset,
-                tolerance);
-  }
-}
-
-}  // namespace
-}  // namespace mediapipe

mediapipe/graphs/object_detection_3d/calculators/gl_animation_overlay_calculator.cc

@@ -24,9 +24,9 @@
 #include "mediapipe/framework/port/status.h"
 #include "mediapipe/gpu/gl_calculator_helper.h"
 #include "mediapipe/gpu/shader_util.h"
-#include "mediapipe/graphs/object_detection_3d/calculators/camera_parameters.pb.h"
 #include "mediapipe/graphs/object_detection_3d/calculators/gl_animation_overlay_calculator.pb.h"
 #include "mediapipe/graphs/object_detection_3d/calculators/model_matrix.pb.h"
+#include "mediapipe/modules/objectron/calculators/camera_parameters.pb.h"
 
 namespace mediapipe {
 
@@ -626,7 +626,7 @@ void GlAnimationOverlayCalculator::LoadModelMatrices(
 
 ::mediapipe::Status GlAnimationOverlayCalculator::Process(
     CalculatorContext *cc) {
-  return helper_.RunInGlContext([this, &cc]() -> ::mediapipe::Status {
+  return helper_.RunInGlContext([this, &cc]() -> mediapipe::Status {
     if (!initialized_) {
       MP_RETURN_IF_ERROR(GlSetup());
       initialized_ = true;

mediapipe/graphs/object_detection_3d/calculators/landmarks_to_frame_annotation_calculator.cc

@@ -1,76 +0,0 @@
-// Copyright 2020 The MediaPipe Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-
-#include "absl/memory/memory.h"
-#include "mediapipe/framework/calculator_framework.h"
-#include "mediapipe/framework/formats/landmark.pb.h"
-#include "mediapipe/framework/port/ret_check.h"
-#include "mediapipe/framework/port/status.h"
-#include "mediapipe/graphs/object_detection_3d/calculators/annotation_data.pb.h"
-
-namespace mediapipe {
-
-namespace {
-
-constexpr char kInputLandmarksTag[] = "LANDMARKS";
-constexpr char kOutputFrameAnnotationTag[] = "FRAME_ANNOTATION";
-
-}  // namespace
-
-// A calculator that converts NormalizedLandmarkList to FrameAnnotation proto.
-class LandmarksToFrameAnnotationCalculator : public CalculatorBase {
- public:
-  static ::mediapipe::Status GetContract(CalculatorContract* cc);
-  ::mediapipe::Status Process(CalculatorContext* cc) override;
-};
-REGISTER_CALCULATOR(LandmarksToFrameAnnotationCalculator);
-
-::mediapipe::Status LandmarksToFrameAnnotationCalculator::GetContract(
-    CalculatorContract* cc) {
-  RET_CHECK(!cc->Inputs().GetTags().empty());
-  RET_CHECK(!cc->Outputs().GetTags().empty());
-
-  if (cc->Inputs().HasTag(kInputLandmarksTag)) {
-    cc->Inputs().Tag(kInputLandmarksTag).Set<NormalizedLandmarkList>();
-  }
-
-  if (cc->Outputs().HasTag(kOutputFrameAnnotationTag)) {
-    cc->Outputs().Tag(kOutputFrameAnnotationTag).Set<FrameAnnotation>();
-  }
-  return ::mediapipe::OkStatus();
-}
-
-::mediapipe::Status LandmarksToFrameAnnotationCalculator::Process(
-    CalculatorContext* cc) {
-  auto frame_annotation = absl::make_unique<FrameAnnotation>();
-  auto* box_annotation = frame_annotation->add_annotations();
-
-  const auto& landmarks =
-      cc->Inputs().Tag(kInputLandmarksTag).Get<NormalizedLandmarkList>();
-  RET_CHECK_GT(landmarks.landmark_size(), 0)
-      << "Input landmark vector is empty.";
-  for (int i = 0; i < landmarks.landmark_size(); ++i) {
-    auto* point2d = box_annotation->add_keypoints()->mutable_point_2d();
-    point2d->set_x(landmarks.landmark(i).x());
-    point2d->set_y(landmarks.landmark(i).y());
-  }
-  // Output
-  if (cc->Outputs().HasTag(kOutputFrameAnnotationTag)) {
-    cc->Outputs()
-        .Tag(kOutputFrameAnnotationTag)
-        .Add(frame_annotation.release(), cc->InputTimestamp());
-  }
-  return ::mediapipe::OkStatus();
-}
-
-}  // namespace mediapipe

mediapipe/graphs/object_detection_3d/calculators/lift_2d_frame_annotation_to_3d_calculator.cc

@@ -1,168 +0,0 @@
-// Copyright 2020 The MediaPipe Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include <memory>
-#include <unordered_map>
-#include <vector>
-
-#include "Eigen/Dense"
-#include "absl/memory/memory.h"
-#include "absl/strings/str_format.h"
-#include "absl/types/span.h"
-#include "mediapipe/framework/calculator_framework.h"
-#include "mediapipe/framework/deps/file_path.h"
-#include "mediapipe/framework/port/ret_check.h"
-#include "mediapipe/graphs/object_detection_3d/calculators/annotation_data.pb.h"
-#include "mediapipe/graphs/object_detection_3d/calculators/decoder.h"
-#include "mediapipe/graphs/object_detection_3d/calculators/lift_2d_frame_annotation_to_3d_calculator.pb.h"
-#include "mediapipe/graphs/object_detection_3d/calculators/tensor_util.h"
-
-namespace {
-constexpr char kInputStreamTag[] = "FRAME_ANNOTATION";
-constexpr char kOutputStreamTag[] = "LIFTED_FRAME_ANNOTATION";
-
-// Each detection object will be assigned an unique id that starts from 1.
-static int object_id = 0;
-
-inline int GetNextObjectId() { return ++object_id; }
-}  // namespace
-
-namespace mediapipe {
-
-// Lifted the 2D points in a tracked frame annotation to 3D.
-//
-// Input:
-//  FRAME_ANNOTATIONS - Frame annotations with detected 2D points
-// Output:
-//  LIFTED_FRAME_ANNOTATIONS - Result FrameAnnotation with lifted 3D points.
-//
-// Usage example:
-// node {
-//   calculator: "Lift2DFrameAnnotationTo3DCalculator"
-//   input_stream: "FRAME_ANNOTATIONS:tracked_annotations"
-//   output_stream: "LIFTED_FRAME_ANNOTATIONS:lifted_3d_annotations"
-// }
-class Lift2DFrameAnnotationTo3DCalculator : public CalculatorBase {
- public:
-  static ::mediapipe::Status GetContract(CalculatorContract* cc);
-
-  ::mediapipe::Status Open(CalculatorContext* cc) override;
-  ::mediapipe::Status Process(CalculatorContext* cc) override;
-  ::mediapipe::Status Close(CalculatorContext* cc) override;
-
- private:
-  ::mediapipe::Status ProcessCPU(CalculatorContext* cc,
-                                 FrameAnnotation* output_objects);
-  ::mediapipe::Status LoadOptions(CalculatorContext* cc);
-
-  // Increment and assign object ID for each detected object.
-  // In a single MediaPipe session, the IDs are unique.
-  // Also assign timestamp for the FrameAnnotation to be the input packet
-  // timestamp.
-  void AssignObjectIdAndTimestamp(int64 timestamp_us,
-                                  FrameAnnotation* annotation);
-  std::unique_ptr<Decoder> decoder_;
-  ::mediapipe::Lift2DFrameAnnotationTo3DCalculatorOptions options_;
-  Eigen::Matrix<float, 4, 4, Eigen::RowMajor> projection_matrix_;
-};
-REGISTER_CALCULATOR(Lift2DFrameAnnotationTo3DCalculator);
-
-::mediapipe::Status Lift2DFrameAnnotationTo3DCalculator::GetContract(
-    CalculatorContract* cc) {
-  RET_CHECK(cc->Inputs().HasTag(kInputStreamTag));
-  RET_CHECK(cc->Outputs().HasTag(kOutputStreamTag));
-  cc->Inputs().Tag(kInputStreamTag).Set<FrameAnnotation>();
-  cc->Outputs().Tag(kOutputStreamTag).Set<FrameAnnotation>();
-
-  return ::mediapipe::OkStatus();
-}
-
-::mediapipe::Status Lift2DFrameAnnotationTo3DCalculator::Open(
-    CalculatorContext* cc) {
-  MP_RETURN_IF_ERROR(LoadOptions(cc));
-  // clang-format off
-  projection_matrix_ <<
-      1.5731,   0,       0,    0,
-      0,   2.0975,       0,    0,
-      0,        0, -1.0002, -0.2,
-      0,        0,      -1,    0;
-  // clang-format on
-
-  decoder_ = absl::make_unique<Decoder>(
-      BeliefDecoderConfig(options_.decoder_config()));
-  return ::mediapipe::OkStatus();
-}
-
-::mediapipe::Status Lift2DFrameAnnotationTo3DCalculator::Process(
-    CalculatorContext* cc) {
-  if (cc->Inputs().Tag(kInputStreamTag).IsEmpty()) {
-    return ::mediapipe::OkStatus();
-  }
-
-  auto output_objects = absl::make_unique<FrameAnnotation>();
-
-  MP_RETURN_IF_ERROR(ProcessCPU(cc, output_objects.get()));
-
-  // Output
-  if (cc->Outputs().HasTag(kOutputStreamTag)) {
-    cc->Outputs()
-        .Tag(kOutputStreamTag)
-        .Add(output_objects.release(), cc->InputTimestamp());
-  }
-
-  return ::mediapipe::OkStatus();
-}
-
-::mediapipe::Status Lift2DFrameAnnotationTo3DCalculator::ProcessCPU(
-    CalculatorContext* cc, FrameAnnotation* output_objects) {
-  const auto& input_frame_annotations =
-      cc->Inputs().Tag(kInputStreamTag).Get<FrameAnnotation>();
-  // Copy the input frame annotation to the output
-  *output_objects = input_frame_annotations;
-
-  auto status = decoder_->Lift2DTo3D(projection_matrix_, /*portrait*/ true,
-                                     output_objects);
-  if (!status.ok()) {
-    LOG(ERROR) << status;
-    return status;
-  }
-  AssignObjectIdAndTimestamp(cc->InputTimestamp().Microseconds(),
-                             output_objects);
-
-  return ::mediapipe::OkStatus();
-}
-
-::mediapipe::Status Lift2DFrameAnnotationTo3DCalculator::Close(
-    CalculatorContext* cc) {
-  return ::mediapipe::OkStatus();
-}
-
-::mediapipe::Status Lift2DFrameAnnotationTo3DCalculator::LoadOptions(
-    CalculatorContext* cc) {
-  // Get calculator options specified in the graph.
-  options_ =
-      cc->Options<::mediapipe::Lift2DFrameAnnotationTo3DCalculatorOptions>();
-
-  return ::mediapipe::OkStatus();
-}
-
-void Lift2DFrameAnnotationTo3DCalculator::AssignObjectIdAndTimestamp(
-    int64 timestamp_us, FrameAnnotation* annotation) {
-  for (auto& ann : *annotation->mutable_annotations()) {
-    ann.set_object_id(GetNextObjectId());
-  }
-  annotation->set_timestamp(timestamp_us);
-}
-
-}  // namespace mediapipe

mediapipe/graphs/object_detection_3d/calculators/lift_2d_frame_annotation_to_3d_calculator.proto

@@ -1,30 +0,0 @@
-// Copyright 2020 The MediaPipe Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// The option proto for the Lift2DFrameAnnotationTo3DCalculatorOptions.
-
-syntax = "proto2";
-
-package mediapipe;
-
-import "mediapipe/framework/calculator.proto";
-import "mediapipe/graphs/object_detection_3d/calculators/belief_decoder_config.proto";
-
-message Lift2DFrameAnnotationTo3DCalculatorOptions {
-  extend CalculatorOptions {
-    optional Lift2DFrameAnnotationTo3DCalculatorOptions ext = 290166284;
-  }
-
-  optional BeliefDecoderConfig decoder_config = 1;
-}

mediapipe/graphs/object_detection_3d/calculators/model.cc

@@ -1,101 +0,0 @@
-// Copyright 2020 The MediaPipe Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "mediapipe/graphs/object_detection_3d/calculators/model.h"
-
-#include "mediapipe/framework/port/logging.h"
-
-namespace mediapipe {
-
-void Model::SetTransformation(const Eigen::Matrix4f& transform) {
-  transformation_ = transform;
-}
-
-void Model::SetTranslation(const Eigen::Vector3f& translation) {
-  transformation_.col(3).template head<3>() = translation;
-}
-
-void Model::SetRotation(float roll, float pitch, float yaw) {
-  // In our coordinate system, Y is up. We first rotate the object around Y
-  // (yaw), then around Z (pitch), and finally around X (roll).
-  Eigen::Matrix3f r;
-  r = Eigen::AngleAxisf(yaw, Eigen::Vector3f::UnitY()) *
-      Eigen::AngleAxisf(pitch, Eigen::Vector3f::UnitZ()) *
-      Eigen::AngleAxisf(roll, Eigen::Vector3f::UnitX());
-  transformation_.topLeftCorner<3, 3>() = r;
-}
-
-void Model::SetRotation(const Eigen::Matrix3f& rotation) {
-  transformation_.topLeftCorner<3, 3>() = rotation;
-}
-
-void Model::SetScale(const Eigen::Vector3f& scale) { scale_ = scale; }
-
-void Model::SetCategory(const std::string& category) { category_ = category; }
-
-const Eigen::Vector3f Model::GetRotationAngles() const {
-  Vector3f ypr = transformation_.topLeftCorner<3, 3>().eulerAngles(1, 2, 0);
-  return Vector3f(ypr(2), ypr(1), ypr(0));  // swap YPR with RPY
-}
-
-const Eigen::Matrix4f& Model::GetTransformation() const {
-  return transformation_;
-}
-
-const Eigen::Vector3f& Model::GetScale() const { return scale_; }
-
-const Eigen::Ref<const Eigen::Vector3f> Model::GetTranslation() const {
-  return transformation_.col(3).template head<3>();
-}
-
-const Eigen::Ref<const Eigen::Matrix3f> Model::GetRotation() const {
-  return transformation_.template topLeftCorner<3, 3>();
-}
-
-const std::string& Model::GetCategory() const { return category_; }
-
-void Model::Deserialize(const Object& obj) {
-  CHECK_EQ(obj.rotation_size(), 9);
-  CHECK_EQ(obj.translation_size(), 3);
-  CHECK_EQ(obj.scale_size(), 3);
-  category_ = obj.category();
-
-  using RotationMatrix = Eigen::Matrix<float, 3, 3, Eigen::RowMajor>;
-  transformation_.setIdentity();
-  transformation_.topLeftCorner<3, 3>() =
-      Eigen::Map<const RotationMatrix>(obj.rotation().data());
-  transformation_.col(3).head<3>() =
-      Eigen::Map<const Eigen::Vector3f>(obj.translation().data());
-  scale_ = Eigen::Map<const Eigen::Vector3f>(obj.scale().data());
-  Update();
-}
-
-void Model::Serialize(Object* obj) {
-  obj->set_category(category_);
-  for (int i = 0; i < 3; ++i) {
-    for (int j = 0; j < 3; ++j) {
-      obj->add_rotation(transformation_(i, j));
-    }
-  }
-
-  for (int i = 0; i < 3; ++i) {
-    obj->add_translation(transformation_(i, 3));
-  }
-
-  for (int i = 0; i < 3; ++i) {
-    obj->add_scale(scale_[i]);
-  }
-}
-
-}  // namespace mediapipe

mediapipe/graphs/object_detection_3d/calculators/model.h

@@ -1,92 +0,0 @@
-// Copyright 2020 The MediaPipe Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#ifndef MEDIAPIPE_GRAPHS_OBJECT_DETECTION_3D_MODEL_H_
-#define MEDIAPIPE_GRAPHS_OBJECT_DETECTION_3D_MODEL_H_
-
-#include "mediapipe/graphs/object_detection_3d/calculators/annotation_data.pb.h"
-#include "mediapipe/graphs/object_detection_3d/calculators/object.pb.h"
-#include "mediapipe/graphs/object_detection_3d/calculators/types.h"
-
-namespace mediapipe {
-
-class Model {
- public:
-  EIGEN_MAKE_ALIGNED_OPERATOR_NEW
-
-  enum Type {
-    kVisualizationOnly = 0,
-    kBoundingBox,
-    kSkeleton,
-    kShape,  // Shape is a virtual object.
-    kNumModes,
-  };
-
-  virtual ~Model() = default;
-
-  virtual void SetTransformation(const Eigen::Matrix4f& transform);
-  virtual void SetTranslation(const Eigen::Vector3f& translation);
-
-  // Compute the rotation matrix from these angles and update the transformation
-  // matrix accordingly
-  virtual void SetRotation(float roll, float pitch, float yaw);
-  virtual void SetRotation(const Eigen::Matrix3f& rotation);
-  virtual void SetScale(const Eigen::Vector3f& scale);
-  virtual void SetCategory(const std::string& category);
-  virtual size_t GetNumberKeypoints() const { return number_keypoints_; }
-
-  // Gets Euler angles in the order of roll, pitch, yaw.
-  virtual const Eigen::Vector3f GetRotationAngles() const;
-  virtual const Eigen::Matrix4f& GetTransformation() const;
-  virtual const Eigen::Vector3f& GetScale() const;
-  virtual const Eigen::Ref<const Eigen::Vector3f> GetTranslation() const;
-  virtual const Eigen::Ref<const Eigen::Matrix3f> GetRotation() const;
-  virtual const std::string& GetCategory() const;
-
-  // Update the model's keypoints in the world-coordinate system.
-  // The update includes transforming the model to the world-coordinate system
-  // as well as scaling the model.
-  // The user is expected to call this function after Setting the rotation,
-  // orientation or the scale of the model to get an updated model.
-  virtual void Update() = 0;
-
-  // Update the model's parameters (orientation, position, and scale) from the
-  // user-provided variables.
-  virtual void Adjust(const std::vector<float>& variables) = 0;
-
-  // Returns a pointer to the model's keypoints.
-  // Use Eigen::Map to cast the pointer back to Vector3 or Vector4
-  virtual const float* GetVertex(size_t id) const = 0;
-  virtual float* GetVertex(size_t id) = 0;
-  virtual void Deserialize(const Object& obj);
-  virtual void Serialize(Object* obj);
-
-  // TODO: make member variables protected, and add public apis.
-  // 4x4 transformation matrix mapping the first keypoint to world coordinate
-  Eigen::Matrix4f transformation_;
-  Eigen::Vector3f scale_;  // width, height, depth
-  Type model_type_;
-  size_t number_keypoints_;
-  std::string category_;
-
- protected:
-  Model(Type type, size_t number_keypoints, const std::string& category)
-      : model_type_(type),
-        number_keypoints_(number_keypoints),
-        category_(category) {}
-};
-
-}  // namespace mediapipe
-
-#endif  // MEDIAPIPE_GRAPHS_OBJECT_DETECTION_3D_MODEL_H_

mediapipe/graphs/object_detection_3d/calculators/object.proto

@@ -1,124 +0,0 @@
-// Copyright 2020 The MediaPipe Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-syntax = "proto3";
-
-package mediapipe;
-
-message KeyPoint {
-  // The position of the keypoint in the local coordinate system of the rigid
-  // object.
-  float x = 1;
-  float y = 2;
-  float z = 3;
-
-  // Sphere around the keypoint, indiciating annotator's confidence of the
-  // position in meters.
-  float confidence_radius = 4;
-
-  // The name of the keypoint (e.g. legs, head, etc.).
-  // Does not have to be unique.
-  string name = 5;
-
-  // Indicates whether the keypoint is hidden or not.
-  bool hidden = 6;
-}
-
-message Object {
-  // Unique object id through a sequence. There might be multiple objects of
-  // the same label in this sequence.
-  int32 id = 1;
-
-  // Describes what category an object is. E.g. object class, attribute,
-  // instance or person identity. This provides additional context for the
-  // object type.
-  string category = 2;
-
-  enum Type {
-    UNDEFINED_TYPE = 0;
-    BOUNDING_BOX = 1;
-    SKELETON = 2;
-  }
-
-  Type type = 3;
-
-  // 3x3 row-major rotation matrix describing the orientation of the rigid
-  // object's frame of reference in the world-coordinate system.
-  repeated float rotation = 4;
-
-  // 3x1 vector describing the translation of the rigid object's frame of
-  // reference in the world-coordinate system in meters.
-  repeated float translation = 5;
-
-  // 3x1 vector describing the scale of the rigid object's frame of reference in
-  // the world-coordinate system in meters.
-  repeated float scale = 6;
-
-  // List of all the key points associated with this object in the object
-  // coordinate system.
-  // The first keypoint is always the object's frame of reference,
-  // e.g. the centroid of the box.
-  // E.g. bounding box with its center as frame of reference, the 9 keypoints :
-  // {0., 0., 0.},
-  // {-.5, -.5, -.5}, {-.5, -.5, +.5}, {-.5, +.5, -.5}, {-.5, +.5, +.5},
-  // {+.5, -.5, -.5}, {+.5, -.5, +.5}, {+.5, +.5, -.5}, {+.5, +.5, +.5}
-  // To get the bounding box in the world-coordinate system, we first scale the
-  // box then transform the scaled box.
-  // For example, bounding box in the world coordinate system is
-  // rotation * scale * keypoints + translation
-  repeated KeyPoint keypoints = 7;
-
-  // Enum to reflect how this object is created.
-  enum Method {
-    UNKNOWN_METHOD = 0;
-    ANNOTATION = 1;    // Created by data annotation.
-    AUGMENTATION = 2;  // Created by data augmentation.
-  }
-  Method method = 8;
-}
-
-// The edge connecting two keypoints together
-message Edge {
-  // keypoint id of the edge's source
-  int32 source = 1;
-
-  // keypoint id of the edge's sink
-  int32 sink = 2;
-}
-
-// The skeleton template for different objects (e.g. humans, chairs, hands, etc)
-// The annotation tool reads the skeleton template dictionary.
-message Skeleton {
-  // The origin keypoint in the object coordinate system. (i.e. Point 0, 0, 0)
-  int32 reference_keypoint = 1;
-
-  // The skeleton's category (e.g. human, chair, hand.). Should be unique in the
-  // dictionary.
-  string category = 2;
-
-  // Initialization value for all the keypoints in the skeleton in the object's
-  // local coordinate system. Pursuit will transform these points using object's
-  // transformation to get the keypoint in the world-cooridnate.
-  repeated KeyPoint keypoints = 3;
-
-  // List of edges connecting keypoints
-  repeated Edge edges = 4;
-}
-
-// The list of all the modeled skeletons in our library. These models can be
-// objects (chairs, desks, etc), humans (full pose, hands, faces, etc), or box.
-// We can have multiple skeletons in the same file.
-message Skeletons {
-  repeated Skeleton object = 1;
-}

mediapipe/graphs/object_detection_3d/calculators/tensor_util.cc

@@ -1,33 +0,0 @@
-// Copyright 2020 The MediaPipe Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "mediapipe/graphs/object_detection_3d/calculators/tensor_util.h"
-
-#include "mediapipe/framework/port/logging.h"
-
-namespace mediapipe {
-
-cv::Mat ConvertTfliteTensorToCvMat(const TfLiteTensor& tensor) {
-  // Check tensor is BxCxWxH (size = 4) and the batch size is one(data[0] = 1)
-  CHECK(tensor.dims->size == 4 && tensor.dims->data[0] == 1);
-  CHECK_EQ(kTfLiteFloat32, tensor.type) << "tflite_tensor type is not float";
-
-  const size_t num_output_channels = tensor.dims->data[3];
-  const int dims = 2;
-  const int sizes[] = {tensor.dims->data[1], tensor.dims->data[2]};
-  const int type = CV_MAKETYPE(CV_32F, num_output_channels);
-  return cv::Mat(dims, sizes, type, reinterpret_cast<void*>(tensor.data.f));
-}
-
-}  // namespace mediapipe

mediapipe/graphs/object_detection_3d/calculators/tensor_util.h

@@ -1,27 +0,0 @@
-// Copyright 2020 The MediaPipe Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#ifndef MEDIAPIPE_GRAPHS_OBJECT_DETECTION_3D_TENSOR_UTIL_H_
-#define MEDIAPIPE_GRAPHS_OBJECT_DETECTION_3D_TENSOR_UTIL_H_
-
-#include "mediapipe/framework/port/opencv_core_inc.h"
-#include "tensorflow/lite/interpreter.h"
-
-namespace mediapipe {
-
-// Converts a single channel tflite tensor to a grayscale image
-cv::Mat ConvertTfliteTensorToCvMat(const TfLiteTensor& tensor);
-}  // namespace mediapipe
-
-#endif  // MEDIAPIPE_GRAPHS_OBJECT_DETECTION_3D_TENSOR_UTIL_H_

mediapipe/graphs/object_detection_3d/calculators/tflite_tensors_to_objects_calculator.cc

@@ -1,216 +0,0 @@
-// Copyright 2020 The MediaPipe Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include <memory>
-#include <unordered_map>
-#include <vector>
-
-#include "Eigen/Dense"
-#include "absl/memory/memory.h"
-#include "absl/strings/str_format.h"
-#include "absl/types/span.h"
-#include "mediapipe/framework/calculator_framework.h"
-#include "mediapipe/framework/deps/file_path.h"
-#include "mediapipe/framework/port/opencv_core_inc.h"
-#include "mediapipe/framework/port/ret_check.h"
-#include "mediapipe/graphs/object_detection_3d/calculators/annotation_data.pb.h"
-#include "mediapipe/graphs/object_detection_3d/calculators/belief_decoder_config.pb.h"
-#include "mediapipe/graphs/object_detection_3d/calculators/decoder.h"
-#include "mediapipe/graphs/object_detection_3d/calculators/tensor_util.h"
-#include "mediapipe/graphs/object_detection_3d/calculators/tflite_tensors_to_objects_calculator.pb.h"
-#include "tensorflow/lite/interpreter.h"
-
-namespace {
-constexpr char kInputStreamTag[] = "TENSORS";
-constexpr char kOutputStreamTag[] = "ANNOTATIONS";
-
-// Each detection object will be assigned an unique id that starts from 1.
-static int object_id = 0;
-
-inline int GetNextObjectId() { return ++object_id; }
-}  // namespace
-
-namespace mediapipe {
-
-// Convert result TFLite tensors from deep pursuit 3d model into
-// FrameAnnotation.
-//
-// Input:
-//  TENSORS - Vector of TfLiteTensor of type kTfLiteFloat32.
-// Output:
-//  ANNOTATIONS - Result FrameAnnotation.
-//
-// Usage example:
-// node {
-//   calculator: "TfLiteTensorsToObjectsCalculator"
-//   input_stream: "TENSORS:tensors"
-//   output_stream: "ANNOTATIONS:annotations"
-// }
-class TfLiteTensorsToObjectsCalculator : public CalculatorBase {
- public:
-  static ::mediapipe::Status GetContract(CalculatorContract* cc);
-
-  ::mediapipe::Status Open(CalculatorContext* cc) override;
-  ::mediapipe::Status Process(CalculatorContext* cc) override;
-  ::mediapipe::Status Close(CalculatorContext* cc) override;
-
- private:
-  ::mediapipe::Status ProcessCPU(CalculatorContext* cc,
-                                 FrameAnnotation* output_objects);
-  ::mediapipe::Status LoadOptions(CalculatorContext* cc);
-  // Takes point_3d in FrameAnnotation, projects to 2D, and overwrite the
-  // point_2d field with the projection.
-  void Project3DTo2D(bool portrait, FrameAnnotation* annotation) const;
-  // Increment and assign object ID for each detected object.
-  // In a single MediaPipe session, the IDs are unique.
-  // Also assign timestamp for the FrameAnnotation to be the input packet
-  // timestamp.
-  void AssignObjectIdAndTimestamp(int64 timestamp_us,
-                                  FrameAnnotation* annotation);
-
-  int num_classes_ = 0;
-  int num_keypoints_ = 0;
-
-  ::mediapipe::TfLiteTensorsToObjectsCalculatorOptions options_;
-  std::unique_ptr<Decoder> decoder_;
-  Eigen::Matrix<float, 4, 4, Eigen::RowMajor> projection_matrix_;
-};
-REGISTER_CALCULATOR(TfLiteTensorsToObjectsCalculator);
-
-::mediapipe::Status TfLiteTensorsToObjectsCalculator::GetContract(
-    CalculatorContract* cc) {
-  RET_CHECK(!cc->Inputs().GetTags().empty());
-  RET_CHECK(!cc->Outputs().GetTags().empty());
-
-  if (cc->Inputs().HasTag(kInputStreamTag)) {
-    cc->Inputs().Tag(kInputStreamTag).Set<std::vector<TfLiteTensor>>();
-  }
-
-  if (cc->Outputs().HasTag(kOutputStreamTag)) {
-    cc->Outputs().Tag(kOutputStreamTag).Set<FrameAnnotation>();
-  }
-  return ::mediapipe::OkStatus();
-}
-
-::mediapipe::Status TfLiteTensorsToObjectsCalculator::Open(
-    CalculatorContext* cc) {
-  MP_RETURN_IF_ERROR(LoadOptions(cc));
-  // clang-format off
-  projection_matrix_ <<
-      1.5731,   0,       0, 0,
-      0,   2.0975,       0, 0,
-      0,        0, -1.0002, -0.2,
-      0,        0,      -1, 0;
-  // clang-format on
-  decoder_ = absl::make_unique<Decoder>(
-      BeliefDecoderConfig(options_.decoder_config()));
-
-  return ::mediapipe::OkStatus();
-}
-
-::mediapipe::Status TfLiteTensorsToObjectsCalculator::Process(
-    CalculatorContext* cc) {
-  if (cc->Inputs().Tag(kInputStreamTag).IsEmpty()) {
-    return ::mediapipe::OkStatus();
-  }
-
-  auto output_objects = absl::make_unique<FrameAnnotation>();
-
-  MP_RETURN_IF_ERROR(ProcessCPU(cc, output_objects.get()));
-
-  // Output
-  if (cc->Outputs().HasTag(kOutputStreamTag)) {
-    cc->Outputs()
-        .Tag(kOutputStreamTag)
-        .Add(output_objects.release(), cc->InputTimestamp());
-  }
-
-  return ::mediapipe::OkStatus();
-}
-
-::mediapipe::Status TfLiteTensorsToObjectsCalculator::ProcessCPU(
-    CalculatorContext* cc, FrameAnnotation* output_objects) {
-  const auto& input_tensors =
-      cc->Inputs().Tag(kInputStreamTag).Get<std::vector<TfLiteTensor>>();
-
-  cv::Mat prediction_heatmap = ConvertTfliteTensorToCvMat(input_tensors[0]);
-  cv::Mat offsetmap = ConvertTfliteTensorToCvMat(input_tensors[1]);
-
-  *output_objects =
-      decoder_->DecodeBoundingBoxKeypoints(prediction_heatmap, offsetmap);
-  auto status = decoder_->Lift2DTo3D(projection_matrix_, /*portrait*/ true,
-                                     output_objects);
-  if (!status.ok()) {
-    LOG(ERROR) << status;
-    return status;
-  }
-  Project3DTo2D(/*portrait*/ true, output_objects);
-  AssignObjectIdAndTimestamp(cc->InputTimestamp().Microseconds(),
-                             output_objects);
-
-  return ::mediapipe::OkStatus();
-}
-
-::mediapipe::Status TfLiteTensorsToObjectsCalculator::Close(
-    CalculatorContext* cc) {
-  return ::mediapipe::OkStatus();
-}
-
-::mediapipe::Status TfLiteTensorsToObjectsCalculator::LoadOptions(
-    CalculatorContext* cc) {
-  // Get calculator options specified in the graph.
-  options_ =
-      cc->Options<::mediapipe::TfLiteTensorsToObjectsCalculatorOptions>();
-
-  num_classes_ = options_.num_classes();
-  num_keypoints_ = options_.num_keypoints();
-
-  // Currently only support 2D when num_values_per_keypoint equals to 2.
-  CHECK_EQ(options_.num_values_per_keypoint(), 2);
-
-  return ::mediapipe::OkStatus();
-}
-
-void TfLiteTensorsToObjectsCalculator::Project3DTo2D(
-    bool portrait, FrameAnnotation* annotation) const {
-  for (auto& ann : *annotation->mutable_annotations()) {
-    for (auto& key_point : *ann.mutable_keypoints()) {
-      Eigen::Vector4f point3d;
-      point3d << key_point.point_3d().x(), key_point.point_3d().y(),
-          key_point.point_3d().z(), 1.0f;
-      Eigen::Vector4f point3d_projection = projection_matrix_ * point3d;
-      float u, v;
-      const float inv_w = 1.0f / point3d_projection(3);
-      if (portrait) {
-        u = (point3d_projection(1) * inv_w + 1.0f) * 0.5f;
-        v = (point3d_projection(0) * inv_w + 1.0f) * 0.5f;
-      } else {
-        u = (point3d_projection(0) * inv_w + 1.0f) * 0.5f;
-        v = (1.0f - point3d_projection(1) * inv_w) * 0.5f;
-      }
-      key_point.mutable_point_2d()->set_x(u);
-      key_point.mutable_point_2d()->set_y(v);
-    }
-  }
-}
-
-void TfLiteTensorsToObjectsCalculator::AssignObjectIdAndTimestamp(
-    int64 timestamp_us, FrameAnnotation* annotation) {
-  for (auto& ann : *annotation->mutable_annotations()) {
-    ann.set_object_id(GetNextObjectId());
-  }
-  annotation->set_timestamp(timestamp_us);
-}
-
-}  // namespace mediapipe

mediapipe/graphs/object_detection_3d/calculators/tflite_tensors_to_objects_calculator.proto

@@ -1,39 +0,0 @@
-// Copyright 2020 The MediaPipe Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// The option proto for the TfLiteTensorsToObjectsCalculatorOptions.
-
-syntax = "proto2";
-
-package mediapipe;
-
-import "mediapipe/framework/calculator.proto";
-import "mediapipe/graphs/object_detection_3d/calculators/belief_decoder_config.proto";
-
-message TfLiteTensorsToObjectsCalculatorOptions {
-  extend CalculatorOptions {
-    optional TfLiteTensorsToObjectsCalculatorOptions ext = 263667646;
-  }
-
-  // The number of output classes predicted by the detection model.
-  optional int32 num_classes = 1;
-
-  // The number of predicted keypoints.
-  optional int32 num_keypoints = 2;
-  // The dimension of each keypoint, e.g. number of values predicted for each
-  // keypoint.
-  optional int32 num_values_per_keypoint = 3 [default = 2];
-
-  optional BeliefDecoderConfig decoder_config = 4;
-}

mediapipe/graphs/object_detection_3d/calculators/types.h

@@ -1,56 +0,0 @@
-// Copyright 2020 The MediaPipe Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#ifndef MEDIAPIPE_GRAPHS_OBJECT_DETECTION_3D_TYPES_H_
-#define MEDIAPIPE_GRAPHS_OBJECT_DETECTION_3D_TYPES_H_
-
-#include <array>
-
-#include "Eigen/Geometry"
-
-namespace mediapipe {
-
-using Eigen::Map;
-using Eigen::Vector2f;
-using Eigen::Vector3f;
-using Eigen::Vector4f;
-using Matrix4f_RM = Eigen::Matrix<float, 4, 4, Eigen::RowMajor>;
-using Matrix3f_RM = Eigen::Matrix<float, 3, 3, Eigen::RowMajor>;
-
-using Face = std::array<int, 4>;
-
-struct SuperPoint {
-  enum PointSourceType { kPointCloud = 0, kBoundingBox = 1, kSkeleton = 2 };
-  // The id of the point in the point-cloud
-  int reference_point;
-  // The source of the
-  PointSourceType source;
-  // The id of the point in set of points in current frame
-  int id;
-  // If source is kBoundingBox or kSkeleton, object_id stores the id of which \
-  // object this point belongs to.
-  int object_id;
-  // projected u-v value
-  Vector2f uv;
-  Vector2f pixel;
-  // the 3D point
-  Vector3f point_3d;
-  // Color
-  Eigen::Matrix<unsigned char, 4, 1> color;
-  bool rendered;
-};
-
-}  // namespace mediapipe
-
-#endif  // MEDIAPIPE_GRAPHS_OBJECT_DETECTION_3D_TYPES_H_

mediapipe/graphs/object_detection_3d/object_occlusion_tracking.pbtxt

@@ -2,8 +2,15 @@
 
 # Images coming into and out of the graph.
 input_stream: "input_video"
-input_stream: "input_width"
-input_stream: "input_height"
+input_stream: "WIDTH:input_width"
+input_stream: "HEIGHT:input_height"
+input_side_packet: "LABELS_CSV:allowed_labels"
+input_side_packet: "MODEL_SCALE:model_scale"
+input_side_packet: "MODEL_TRANSFORMATION:model_transformation"
+input_side_packet: "TEXTURE:box_texture"
+input_side_packet: "ANIMATION_ASSET:box_asset_name"
+input_side_packet: "MASK_TEXTURE:obj_texture"
+input_side_packet: "MASK_ASSET:obj_asset_name"
 output_stream: "output_video"
 
 # Throttles the images flowing downstream for flow control. It passes through
@@ -19,7 +26,7 @@ output_stream: "output_video"
 node {
   calculator: "FlowLimiterCalculator"
   input_stream: "input_video"
-  input_stream: "FINISHED:box_rect"
+  input_stream: "FINISHED:lifted_objects"
   input_stream_info: {
     tag_index: "FINISHED"
     back_edge: true
@@ -31,7 +38,7 @@ node {
 node: {
   calculator: "ImageCroppingCalculator"
   input_stream: "IMAGE_GPU:throttled_input_video"
-  output_stream: "IMAGE_GPU:throttled_input_video_4x3"
+  output_stream: "IMAGE_GPU:throttled_input_video_3x4"
   input_stream: "WIDTH:input_width"
   input_stream: "HEIGHT:input_height"
   node_options: {
@@ -41,87 +48,11 @@ node: {
   }
 }
 
-# Caches a box-presence decision fed back from boxLandmarkSubgraph, and upon
-# the arrival of the next input image sends out the cached decision with the
-# timestamp replaced by that of the input image, essentially generating a packet
-# that carries the previous box-presence decision. Note that upon the arrival
-# of the very first input image, an empty packet is sent out to jump start the
-# feedback loop.
 node {
-  calculator: "PreviousLoopbackCalculator"
-  input_stream: "MAIN:throttled_input_video_4x3"
-  input_stream: "LOOP:box_presence"
-  input_stream_info: {
-    tag_index: "LOOP"
-    back_edge: true
-  }
-  output_stream: "PREV_LOOP:prev_box_presence"
-}
-
-# Drops the incoming image if boxLandmarkSubgraph was able to identify box
-# presence in the previous image. Otherwise, passes the incoming image through
-# to trigger a new round of box detection in boxDetectionSubgraph.
-node {
-  calculator: "GateCalculator"
-  input_stream: "throttled_input_video_4x3"
-  input_stream: "DISALLOW:prev_box_presence"
-  output_stream: "detection_input_video"
-
-  node_options: {
-    [type.googleapis.com/mediapipe.GateCalculatorOptions] {
-      empty_packets_as_allow: true
-    }
-  }
-}
-
-# Subgraph that performs 2D object detection.
-node {
-  calculator: "ObjectDetectionOidV4Subgraph"
-  input_stream: "detection_input_video"
-  input_side_packet: "allowed_labels"
-  output_stream: "NORM_RECT:box_rect_from_object_detections"
-}
-
-# Subgraph that localizes box landmarks (see subgraphs/box_landmark_gpu.pbtxt).
-node {
-  calculator: "BoxLandmarkSubgraph"
-  input_stream: "IMAGE:throttled_input_video_4x3"
-  input_stream: "NORM_RECT:box_rect"
+  calculator: "ObjectronGpuSubgraph"
+  input_stream: "IMAGE_GPU:throttled_input_video_3x4"
+  input_side_packet: "LABELS_CSV:allowed_labels"
   output_stream: "FRAME_ANNOTATION:lifted_objects"
-  output_stream: "NORM_RECT:box_rect_from_landmarks"
-  output_stream: "PRESENCE:box_presence"
-}
-
-# Caches a box rectangle fed back from boxLandmarkSubgraph, and upon the
-# arrival of the next input image sends out the cached rectangle with the
-# timestamp replaced by that of the input image, essentially generating a packet
-# that carries the previous box rectangle. Note that upon the arrival of the
-# very first input image, an empty packet is sent out to jump start the
-# feedback loop.
-node {
-  calculator: "PreviousLoopbackCalculator"
-  input_stream: "MAIN:throttled_input_video_4x3"
-  input_stream: "LOOP:box_rect_from_landmarks"
-  input_stream_info: {
-    tag_index: "LOOP"
-    back_edge: true
-  }
-  output_stream: "PREV_LOOP:prev_box_rect_from_landmarks"
-}
-
-# Merges a stream of box rectangles generated by boxDetectionSubgraph and that
-# generated by boxLandmarkSubgraph into a single output stream by selecting
-# between one of the two streams. The former is selected if the incoming packet
-# is not empty, i.e., box detection is performed on the current image by
-# boxDetectionSubgraph (because boxLandmarkSubgraph could not identify box
-# presence in the previous image). Otherwise, the latter is selected, which is
-# never empty because boxLandmarkSubgraphs processes all images (that went
-# through FlowLimiterCaculator).
-node {
-  calculator: "MergeCalculator"
-  input_stream: "box_rect_from_object_detections"
-  input_stream: "prev_box_rect_from_landmarks"
-  output_stream: "box_rect"
 }
 
 # The rendering nodes:
@@ -171,7 +102,7 @@ node {
 # then we render the occlusion mask.
 node: {
   calculator: "GlAnimationOverlayCalculator"
-  input_stream: "VIDEO:throttled_input_video_4x3"
+  input_stream: "VIDEO:throttled_input_video_3x4"
   input_stream: "MODEL_MATRICES:model_matrices"
   input_stream: "MASK_MODEL_MATRICES:mask_model_matrices"
   output_stream: "output_video"

mediapipe/graphs/object_detection_3d/object_occlusion_tracking_1stage.pbtxt

@@ -44,13 +44,13 @@ node {
 }
 
 node {
-  calculator: "ObjectronDetectionSubgraphGpu"
+  calculator: "ObjectronDetection1StageSubgraphGpu"
   input_stream: "IMAGE_GPU:sampled_input_video"
   output_stream: "ANNOTATIONS:objects"
 }
 
 node {
-  calculator: "ObjectronTrackingSubgraphGpu"
+  calculator: "ObjectronTracking1StageSubgraphGpu"
   input_stream: "FRAME_ANNOTATION:objects"
   input_stream: "IMAGE_GPU:input_video_copy"
   output_stream: "LIFTED_FRAME_ANNOTATION:lifted_tracked_objects"

mediapipe/graphs/object_detection_3d/objectron_desktop_cpu.pbtxt

@@ -0,0 +1,60 @@
+# MediaPipe Objectron 3D object detection on Desktop CPU.
+input_side_packet: "INPUT_FILE_PATH:input_video_path"
+input_side_packet: "FILE_PATH:0:box_landmark_model_path"
+input_side_packet: "LABELS_CSV:allowed_labels"
+input_side_packet: "OUTPUT_FILE_PATH:output_video_path"
+
+
+# Decodes an input video file into images and a video header.
+node {
+  calculator: "OpenCvVideoDecoderCalculator"
+  input_side_packet: "INPUT_FILE_PATH:input_video_path"
+  output_stream: "VIDEO:input_video"
+  output_stream: "VIDEO_PRESTREAM:input_video_header"
+}
+
+node {
+  calculator: "LocalFileContentsCalculator"
+  input_side_packet: "FILE_PATH:0:box_landmark_model_path"
+  output_side_packet: "CONTENTS:0:box_landmark_model_blob"
+}
+
+node {
+  calculator: "TfLiteModelCalculator"
+  input_side_packet: "MODEL_BLOB:box_landmark_model_blob"
+  output_side_packet: "MODEL:box_landmark_model"
+}
+
+node {
+  calculator: "ObjectronCpuSubgraph"
+  input_stream: "IMAGE:input_video"
+  input_side_packet: "MODEL:box_landmark_model"
+  input_side_packet: "LABELS_CSV:allowed_labels"
+  output_stream: "LANDMARKS:box_landmarks"
+  output_stream: "NORM_RECT:box_rect"
+}
+
+# Subgraph that renders annotations and overlays them on top of the input
+# images (see renderer_gpu.pbtxt).
+node {
+  calculator: "RendererSubgraph"
+  input_stream: "IMAGE:input_video"
+  input_stream: "LANDMARKS:box_landmarks"
+  input_stream: "NORM_RECT:box_rect"
+  output_stream: "IMAGE:output_video"
+}
+
+# Encodes the annotated images into a video file, adopting properties specified
+# in the input video header, e.g., video framerate.
+node {
+  calculator: "OpenCvVideoEncoderCalculator"
+  input_stream: "VIDEO:output_video"
+  input_stream: "VIDEO_PRESTREAM:input_video_header"
+  input_side_packet: "OUTPUT_FILE_PATH:output_video_path"
+  node_options: {
+    [type.googleapis.com/mediapipe.OpenCvVideoEncoderCalculatorOptions]: {
+      codec: "avc1"
+      video_format: "mp4"
+    }
+  }
+}

mediapipe/graphs/object_detection_3d/subgraphs/BUILD

@@ -22,76 +22,14 @@ licenses(["notice"])
 package(default_visibility = ["//visibility:public"])
 
 mediapipe_simple_subgraph(
-    name = "objectron_detection_gpu",
-    graph = "objectron_detection_gpu.pbtxt",
-    register_as = "ObjectronDetectionSubgraphGpu",
+    name = "renderer_cpu",
+    graph = "renderer_cpu.pbtxt",
+    register_as = "RendererSubgraph",
+    visibility = ["//visibility:public"],
     deps = [
-        "//mediapipe/calculators/image:image_transformation_calculator",
-        "//mediapipe/calculators/tflite:tflite_converter_calculator",
-        "//mediapipe/calculators/tflite:tflite_custom_op_resolver_calculator",
-        "//mediapipe/calculators/tflite:tflite_inference_calculator",
-        "//mediapipe/graphs/object_detection_3d/calculators:tflite_tensors_to_objects_calculator",
-    ],
-)
-
-mediapipe_simple_subgraph(
-    name = "objectron_tracking_gpu",
-    graph = "objectron_tracking_gpu.pbtxt",
-    register_as = "ObjectronTrackingSubgraphGpu",
-    deps = [
-        "//mediapipe/calculators/image:image_transformation_calculator",
-        "//mediapipe/calculators/video:box_tracker_calculator",
-        "//mediapipe/calculators/video:flow_packager_calculator",
-        "//mediapipe/calculators/video:motion_analysis_calculator",
-        "//mediapipe/framework/stream_handler:sync_set_input_stream_handler",
-        "//mediapipe/gpu:gpu_buffer_to_image_frame_calculator",
-        "//mediapipe/graphs/object_detection_3d/calculators:frame_annotation_to_timed_box_list_calculator",
-        "//mediapipe/graphs/object_detection_3d/calculators:frame_annotation_tracker_calculator",
-        "//mediapipe/graphs/object_detection_3d/calculators:lift_2d_frame_annotation_to_3d_calculator",
-    ],
-)
-
-mediapipe_simple_subgraph(
-    name = "box_landmark_gpu",
-    graph = "box_landmark_gpu.pbtxt",
-    register_as = "BoxLandmarkSubgraph",
-    deps = [
-        "//mediapipe/calculators/core:split_vector_calculator",
-        "//mediapipe/calculators/image:image_cropping_calculator",
-        "//mediapipe/calculators/image:image_properties_calculator",
-        "//mediapipe/calculators/image:image_transformation_calculator",
-        "//mediapipe/calculators/tflite:tflite_converter_calculator",
-        "//mediapipe/calculators/tflite:tflite_custom_op_resolver_calculator",
-        "//mediapipe/calculators/tflite:tflite_inference_calculator",
-        "//mediapipe/calculators/tflite:tflite_tensors_to_floats_calculator",
-        "//mediapipe/calculators/tflite:tflite_tensors_to_landmarks_calculator",
-        "//mediapipe/calculators/util:detections_to_rects_calculator",
-        "//mediapipe/calculators/util:landmark_letterbox_removal_calculator",
-        "//mediapipe/calculators/util:landmark_projection_calculator",
-        "//mediapipe/calculators/util:landmarks_smoothing_calculator",
-        "//mediapipe/calculators/util:landmarks_to_detection_calculator",
-        "//mediapipe/calculators/util:rect_transformation_calculator",
-        "//mediapipe/calculators/util:thresholding_calculator",
-        "//mediapipe/graphs/object_detection_3d/calculators:frame_annotation_to_rect_calculator",
-        "//mediapipe/graphs/object_detection_3d/calculators:landmarks_to_frame_annotation_calculator",
-        "//mediapipe/graphs/object_detection_3d/calculators:lift_2d_frame_annotation_to_3d_calculator",
-    ],
-)
-
-mediapipe_simple_subgraph(
-    name = "object_detection_oid_v4_gpu",
-    graph = "object_detection_oid_v4_gpu.pbtxt",
-    register_as = "ObjectDetectionOidV4Subgraph",
-    deps = [
-        "//mediapipe/calculators/image:image_transformation_calculator",
-        "//mediapipe/calculators/tflite:ssd_anchors_calculator",
-        "//mediapipe/calculators/tflite:tflite_converter_calculator",
-        "//mediapipe/calculators/tflite:tflite_inference_calculator",
-        "//mediapipe/calculators/tflite:tflite_tensors_to_detections_calculator",
-        "//mediapipe/calculators/util:detection_label_id_to_text_calculator",
-        "//mediapipe/calculators/util:detections_to_rects_calculator",
-        "//mediapipe/calculators/util:non_max_suppression_calculator",
-        "//mediapipe/calculators/util:rect_transformation_calculator",
-        "//mediapipe/graphs/object_detection_3d/calculators:filter_detection_calculator",
+        "//mediapipe/calculators/util:annotation_overlay_calculator",
+        "//mediapipe/calculators/util:detections_to_render_data_calculator",
+        "//mediapipe/calculators/util:landmarks_to_render_data_calculator",
+        "//mediapipe/calculators/util:rect_to_render_data_calculator",
     ],
 )

mediapipe/graphs/object_detection_3d/subgraphs/box_landmark_gpu.pbtxt

@@ -1,205 +0,0 @@
-# MediaPipe Box landmark localization subgraph.
-
-type: "BoxLandmarkSubgraph"
-
-input_stream: "IMAGE:input_video"
-input_stream: "NORM_RECT:box_rect"
-output_stream: "FRAME_ANNOTATION:lifted_box"
-output_stream: "NORM_RECT:box_rect_for_next_frame"
-output_stream: "PRESENCE:box_presence"
-
-# Crops the rectangle that contains a box from the input image.
-node {
-  calculator: "ImageCroppingCalculator"
-  input_stream: "IMAGE_GPU:input_video"
-  input_stream: "NORM_RECT:box_rect"
-  output_stream: "IMAGE_GPU:box_image"
-  node_options: {
-    [type.googleapis.com/mediapipe.ImageCroppingCalculatorOptions] {
-      border_mode: BORDER_REPLICATE
-    }
-  }
-}
-
-# Transforms the input image on GPU to a 256x256 image. To scale the input
-# image, the scale_mode option is set to FIT to preserve the aspect ratio,
-# resulting in potential letterboxing in the transformed image.
-node: {
-  calculator: "ImageTransformationCalculator"
-  input_stream: "IMAGE_GPU:box_image"
-  output_stream: "IMAGE_GPU:transformed_box_image"
-  output_stream: "LETTERBOX_PADDING:letterbox_padding"
-  node_options: {
-    [type.googleapis.com/mediapipe.ImageTransformationCalculatorOptions] {
-      output_width: 224
-      output_height: 224
-      scale_mode: FIT
-    }
-  }
-}
-
-# Converts the transformed input image on GPU into an image tensor stored as a
-# TfLiteTensor.
-node {
-  calculator: "TfLiteConverterCalculator"
-  input_stream: "IMAGE_GPU:transformed_box_image"
-  output_stream: "TENSORS_GPU:image_tensor"
-  node_options: {
-    [type.googleapis.com/mediapipe.TfLiteConverterCalculatorOptions] {
-      zero_center: false
-    }
-  }
-}
-
-# Generates a single side packet containing a TensorFlow Lite op resolver that
-# supports custom ops needed by the model used in this graph.
-node {
-  calculator: "TfLiteCustomOpResolverCalculator"
-  output_side_packet: "opresolver"
-}
-
-# Runs a TensorFlow Lite model on GPU that takes an image tensor and outputs a
-# vector of tensors representing, for instance, detection boxes/keypoints and
-# scores.
-node {
-  calculator: "TfLiteInferenceCalculator"
-  input_stream: "TENSORS_GPU:image_tensor"
-  output_stream: "TENSORS:output_tensors"
-  input_side_packet: "CUSTOM_OP_RESOLVER:opresolver"
-  node_options: {
-    [type.googleapis.com/mediapipe.TfLiteInferenceCalculatorOptions] {
-      model_path: "object_detection_3d.tflite"
-      use_gpu: true
-    }
-  }
-}
-
-# Splits a vector of tensors into multiple vectors.
-node {
-  calculator: "SplitTfLiteTensorVectorCalculator"
-  input_stream: "output_tensors"
-  output_stream: "landmark_tensors"
-  output_stream: "box_flag_tensor"
-  node_options: {
-    [type.googleapis.com/mediapipe.SplitVectorCalculatorOptions] {
-      ranges: { begin: 0 end: 1 }
-      ranges: { begin: 1 end: 2 }
-    }
-  }
-}
-
-# Converts the box-flag tensor into a float that represents the confidence
-# score of box presence.
-node {
-  calculator: "TfLiteTensorsToFloatsCalculator"
-  input_stream: "TENSORS:box_flag_tensor"
-  output_stream: "FLOAT:box_presence_score"
-}
-
-# Applies a threshold to the confidence score to determine whether a box is
-# present.
-node {
-  calculator: "ThresholdingCalculator"
-  input_stream: "FLOAT:box_presence_score"
-  output_stream: "FLAG:box_presence"
-  node_options: {
-    [type.googleapis.com/mediapipe.ThresholdingCalculatorOptions] {
-      threshold: 0.99
-    }
-  }
-}
-
-# Decodes the landmark tensors into a list of landmarks, where the landmark
-# coordinates are normalized by the size of the input image to the model.
-node {
-  calculator: "TfLiteTensorsToLandmarksCalculator"
-  input_stream: "TENSORS:landmark_tensors"
-  output_stream: "NORM_LANDMARKS:landmarks"
-  node_options: {
-    [type.googleapis.com/mediapipe.TfLiteTensorsToLandmarksCalculatorOptions] {
-      num_landmarks: 9
-      input_image_width: 224
-      input_image_height: 224
-    }
-  }
-}
-
-# Adjusts landmarks (already normalized to [0.f, 1.f]) on the letterboxed box
-# image (after image transformation with the FIT scale mode) to the
-# corresponding locations on the same image with the letterbox removed (box
-# image before image transformation).
-node {
-  calculator: "LandmarkLetterboxRemovalCalculator"
-  input_stream: "LANDMARKS:landmarks"
-  input_stream: "LETTERBOX_PADDING:letterbox_padding"
-  output_stream: "LANDMARKS:scaled_landmarks"
-}
-
-# Projects the landmarks from the cropped box image to the corresponding
-# locations on the full image before cropping (input to the graph).
-node {
-  calculator: "LandmarkProjectionCalculator"
-  input_stream: "NORM_LANDMARKS:scaled_landmarks"
-  input_stream: "NORM_RECT:box_rect"
-  output_stream: "NORM_LANDMARKS:box_landmarks"
-}
-
-# Extracts image size from the input images.
-node {
-  calculator: "ImagePropertiesCalculator"
-  input_stream: "IMAGE_GPU:input_video"
-  output_stream: "SIZE:image_size"
-}
-
-# Smooth predicted landmarks coordinates.
-node {
-  calculator: "LandmarksSmoothingCalculator"
-  input_stream: "NORM_LANDMARKS:box_landmarks"
-  input_stream: "IMAGE_SIZE:image_size"
-  output_stream: "NORM_FILTERED_LANDMARKS:box_landmarks_filtered"
-  node_options: {
-    [type.googleapis.com/mediapipe.LandmarksSmoothingCalculatorOptions] {
-      velocity_filter: {
-        window_size: 10
-        velocity_scale: 7.5
-      }
-    }
-  }
-}
-
-# Convert box landmarks to frame annotation.
-node {
-  calculator: "LandmarksToFrameAnnotationCalculator"
-  input_stream: "LANDMARKS:box_landmarks_filtered"
-  output_stream: "FRAME_ANNOTATION:box_annotation"
-}
-
-# Lift the 2D landmarks to 3D using EPnP algorithm.
-node {
-  calculator: "Lift2DFrameAnnotationTo3DCalculator"
-  input_stream: "FRAME_ANNOTATION:box_annotation"
-  output_stream: "LIFTED_FRAME_ANNOTATION:lifted_box"
-}
-
-# Get rotated rectangle from lifted box.
-node {
-  calculator: "FrameAnnotationToRectCalculator"
-  input_stream: "FRAME_ANNOTATION:lifted_box"
-  output_stream: "NORM_RECT:rect_from_box"
-}
-
-# Expands the box rectangle so that in the next video frame it's likely to
-# still contain the box even with some motion.
-node {
-  calculator: "RectTransformationCalculator"
-  input_stream: "NORM_RECT:rect_from_box"
-  input_stream: "IMAGE_SIZE:image_size"
-  output_stream: "box_rect_for_next_frame"
-  node_options: {
-    [type.googleapis.com/mediapipe.RectTransformationCalculatorOptions] {
-      scale_x: 1.5
-      scale_y: 1.5
-      square_long: true
-    }
-  }
-}

mediapipe/graphs/object_detection_3d/subgraphs/object_detection_oid_v4_gpu.pbtxt

@@ -1,177 +0,0 @@
-# MediaPipe Objectron object bounding box detection subgraph.
-
-type: "ObjectDetectionSubgraph"
-
-input_stream: "input_video"
-input_side_packet: "allowed_labels"
-output_stream: "NORM_RECT:box_rect_from_object_detections"
-
-# Transforms the input image on GPU to a 320x320 image. To scale the image, by
-# default it uses the STRETCH scale mode that maps the entire input image to the
-# entire transformed image. As a result, image aspect ratio may be changed and
-# objects in the image may be deformed (stretched or squeezed), but the object
-# detection model used in this graph is agnostic to that deformation.
-node: {
-  calculator: "ImageTransformationCalculator"
-  input_stream: "IMAGE_GPU:input_video"
-  output_stream: "IMAGE_GPU:transformed_input_video"
-  node_options: {
-    [type.googleapis.com/mediapipe.ImageTransformationCalculatorOptions] {
-      output_width: 300
-      output_height: 300
-    }
-  }
-}
-
-# Converts the transformed input image on GPU into an image tensor stored as a
-# TfLiteTensor.
-node {
-  calculator: "TfLiteConverterCalculator"
-  input_stream: "IMAGE_GPU:transformed_input_video"
-  output_stream: "TENSORS_GPU:image_tensor"
-}
-
-# Runs a TensorFlow Lite model on GPU that takes an image tensor and outputs a
-# vector of tensors representing, for instance, detection boxes/keypoints and
-# scores.
-node {
-  calculator: "TfLiteInferenceCalculator"
-  input_stream: "TENSORS_GPU:image_tensor"
-  output_stream: "TENSORS_GPU:detection_tensors"
-  node_options: {
-    [type.googleapis.com/mediapipe.TfLiteInferenceCalculatorOptions] {
-      model_path: "object_detection_ssd_mobilenetv2_oidv4_fp16.tflite"
-    }
-  }
-}
-
-# Generates a single side packet containing a vector of SSD anchors based on
-# the specification in the options.
-node {
-  calculator: "SsdAnchorsCalculator"
-  output_side_packet: "anchors"
-  node_options: {
-    [type.googleapis.com/mediapipe.SsdAnchorsCalculatorOptions] {
-      num_layers: 6
-      min_scale: 0.2
-      max_scale: 0.95
-      input_size_height: 300
-      input_size_width: 300
-      anchor_offset_x: 0.5
-      anchor_offset_y: 0.5
-      strides: 16
-      strides: 32
-      strides: 64
-      strides: 128
-      strides: 256
-      strides: 512
-      aspect_ratios: 1.0
-      aspect_ratios: 2.0
-      aspect_ratios: 0.5
-      aspect_ratios: 3.0
-      aspect_ratios: 0.3333
-      reduce_boxes_in_lowest_layer: true
-    }
-  }
-}
-
-# Decodes the detection tensors generated by the TensorFlow Lite model, based on
-# the SSD anchors and the specification in the options, into a vector of
-# detections. Each detection describes a detected object.
-node {
-  calculator: "TfLiteTensorsToDetectionsCalculator"
-  input_stream: "TENSORS_GPU:detection_tensors"
-  input_side_packet: "ANCHORS:anchors"
-  output_stream: "DETECTIONS:detections"
-  node_options: {
-    [type.googleapis.com/mediapipe.TfLiteTensorsToDetectionsCalculatorOptions] {
-      num_classes: 195
-      num_boxes: 1917
-      num_coords: 4
-      ignore_classes: 0
-      sigmoid_score: true
-      apply_exponential_on_box_size: true
-      x_scale: 10.0
-      y_scale: 10.0
-      h_scale: 5.0
-      w_scale: 5.0
-      min_score_thresh: 0.6
-    }
-  }
-}
-
-# Performs non-max suppression to remove excessive detections.
-node {
-  calculator: "NonMaxSuppressionCalculator"
-  input_stream: "detections"
-  output_stream: "suppressed_detections"
-  node_options: {
-    [type.googleapis.com/mediapipe.NonMaxSuppressionCalculatorOptions] {
-      min_suppression_threshold: 0.4
-      max_num_detections: 1
-      overlap_type: INTERSECTION_OVER_UNION
-      return_empty_detections: true
-    }
-  }
-}
-
-# Maps detection label IDs to the corresponding label text. The label map is
-# provided in the label_map_path option.
-node {
-  calculator: "DetectionLabelIdToTextCalculator"
-  input_stream: "suppressed_detections"
-  output_stream: "labeled_detections"
-  node_options: {
-    [type.googleapis.com/mediapipe.DetectionLabelIdToTextCalculatorOptions] {
-      label_map_path: "object_detection_oidv4_labelmap.pbtxt"
-    }
-  }
-}
-
-node {
-  calculator: "FilterDetectionCalculator"
-  input_stream: "DETECTIONS:labeled_detections"
-  output_stream: "DETECTIONS:filtered_detections"
-  input_side_packet: "LABELS_CSV:allowed_labels"
-  node_options: {
-    [type.googleapis.com/mediapipe.FilterDetectionCalculatorOptions]: {
-      min_score: 0.4
-    }
-  }
-}
-
-# Extracts image size from the input images.
-node {
-  calculator: "ImagePropertiesCalculator"
-  input_stream: "IMAGE_GPU:input_video"
-  output_stream: "SIZE:image_size"
-}
-
-# Converts results of box detection into a rectangle (normalized by image size)
-# that encloses the box.
-node {
-  calculator: "DetectionsToRectsCalculator"
-  input_stream: "DETECTIONS:filtered_detections"
-  input_stream: "IMAGE_SIZE:image_size"
-  output_stream: "NORM_RECT:box_rect"
-  node_options: {
-    [type.googleapis.com/mediapipe.DetectionsToRectsCalculatorOptions] {
-      output_zero_rect_for_empty_detections: true
-    }
-  }
-}
-
-# Expands the rectangle that contains the box so that it's likely to cover the
-# entire box.
-node {
-  calculator: "RectTransformationCalculator"
-  input_stream: "NORM_RECT:box_rect"
-  input_stream: "IMAGE_SIZE:image_size"
-  output_stream: "box_rect_from_object_detections"
-  node_options: {
-    [type.googleapis.com/mediapipe.RectTransformationCalculatorOptions] {
-      scale_x: 1.5
-      scale_y: 1.5
-    }
-  }
-}

mediapipe/graphs/object_detection_3d/subgraphs/objectron_detection_gpu.pbtxt

@@ -1,81 +0,0 @@
-# MediaPipe Objectron detection gpu subgraph
-
-type: "ObjectronDetectionSubgraphGpu"
-
-input_stream: "IMAGE_GPU:input_video"
-output_stream: "ANNOTATIONS:objects"
-
-# Transforms the input image on GPU to a 480x640 image. To scale the input
-# image, the scale_mode option is set to FIT to preserve the aspect ratio,
-# resulting in potential letterboxing in the transformed image.
-node: {
-  calculator: "ImageTransformationCalculator"
-  input_stream: "IMAGE_GPU:input_video"
-  output_stream: "IMAGE_GPU:transformed_input_video"
-  node_options: {
-    [type.googleapis.com/mediapipe.ImageTransformationCalculatorOptions] {
-      output_width: 480
-      output_height: 640
-      scale_mode: FIT
-    }
-  }
-}
-
-# Converts the transformed input image on GPU into an image tensor stored as a
-# TfLiteTensor.
-node {
-  calculator: "TfLiteConverterCalculator"
-  input_stream: "IMAGE_GPU:transformed_input_video"
-  output_stream: "TENSORS_GPU:image_tensor"
-}
-
-# Generates a single side packet containing a TensorFlow Lite op resolver that
-# supports custom ops needed by the model used in this graph.
-node {
-  calculator: "TfLiteCustomOpResolverCalculator"
-  output_side_packet: "opresolver"
-  node_options: {
-    [type.googleapis.com/mediapipe.TfLiteCustomOpResolverCalculatorOptions] {
-      use_gpu: true
-    }
-  }
-}
-
-# Runs a TensorFlow Lite model on GPU that takes an image tensor and outputs a
-# vector of tensors representing, for instance, detection boxes/keypoints and
-# scores.
-node {
-  calculator: "TfLiteInferenceCalculator"
-  input_stream: "TENSORS_GPU:image_tensor"
-  output_stream: "TENSORS:detection_tensors"
-  input_side_packet: "CUSTOM_OP_RESOLVER:opresolver"
-  node_options: {
-    [type.googleapis.com/mediapipe.TfLiteInferenceCalculatorOptions] {
-      model_path: "object_detection_3d.tflite"
-    }
-  }
-}
-
-# Decodes the model's output tensor (the heatmap and the distance fields) to 2D
-# keypoints. There are nine 2D keypoints: one center keypoint and eight vertices
-# for the 3D bounding box. The calculator parameters determine's the decoder's
-# sensitivity.
-node {
-  calculator: "TfLiteTensorsToObjectsCalculator"
-  input_stream: "TENSORS:detection_tensors"
-  output_stream: "ANNOTATIONS:objects"
-  node_options: {
-    [type.googleapis.com/mediapipe.TfLiteTensorsToObjectsCalculatorOptions] {
-      num_classes: 1
-      num_keypoints: 9
-      decoder_config {
-        heatmap_threshold: 0.6
-        local_max_distance: 2
-        offset_scale_coef: 1.0
-        voting_radius: 2
-        voting_allowance: 1
-        voting_threshold: 0.2
-      }
-    }
-  }
-}

mediapipe/graphs/object_detection_3d/subgraphs/objectron_tracking_gpu.pbtxt

@@ -1,170 +0,0 @@
-# MediaPipe Objectron tracking gpu subgraph
-
-type: "ObjectronTrackingSubgraphGpu"
-
-input_stream: "FRAME_ANNOTATION:objects"
-input_stream: "IMAGE_GPU:input_video"
-output_stream: "LIFTED_FRAME_ANNOTATION:lifted_tracked_objects"
-
-
-# Converts the detected keypoints to Boxes, used by the tracking subgraph.
-node {
-  calculator: "FrameAnnotationToTimedBoxListCalculator"
-  input_stream: "FRAME_ANNOTATION:objects"
-  output_stream: "BOXES:start_pos"
-}
-
-node: {
-  calculator: "ImageTransformationCalculator"
-  input_stream: "IMAGE_GPU:input_video"
-  output_stream: "IMAGE_GPU:downscaled_input_video"
-  node_options: {
-    [type.googleapis.com/mediapipe.ImageTransformationCalculatorOptions] {
-      output_width: 240
-      output_height: 320
-    }
-  }
-}
-
-# Converts GPU buffer to ImageFrame for processing tracking.
-node: {
-  calculator: "GpuBufferToImageFrameCalculator"
-  input_stream: "downscaled_input_video"
-  output_stream: "downscaled_input_video_cpu"
-}
-
-# Performs motion analysis on an incoming video stream.
-node: {
-  calculator: "MotionAnalysisCalculator"
-  input_stream: "VIDEO:downscaled_input_video_cpu"
-  output_stream: "CAMERA:camera_motion"
-  output_stream: "FLOW:region_flow"
-
-  node_options: {
-    [type.googleapis.com/mediapipe.MotionAnalysisCalculatorOptions]: {
-      analysis_options {
-        analysis_policy: ANALYSIS_POLICY_CAMERA_MOBILE
-        flow_options {
-          fast_estimation_min_block_size: 100
-          top_inlier_sets: 1
-          frac_inlier_error_threshold: 3e-3
-          downsample_mode: DOWNSAMPLE_TO_INPUT_SIZE
-          verification_distance: 5.0
-          verify_long_feature_acceleration: true
-          verify_long_feature_trigger_ratio: 0.1
-          tracking_options {
-            max_features: 500
-            adaptive_extraction_levels: 2
-            min_eig_val_settings {
-              adaptive_lowest_quality_level: 2e-4
-            }
-            klt_tracker_implementation: KLT_OPENCV
-          }
-        }
-      }
-    }
-  }
-}
-
-# Reads optical flow fields defined in
-# mediapipe/framework/formats/motion/optical_flow_field.h,
-# returns a VideoFrame with 2 channels (v_x and v_y), each channel is quantized
-# to 0-255.
-node: {
-  calculator: "FlowPackagerCalculator"
-  input_stream: "FLOW:region_flow"
-  input_stream: "CAMERA:camera_motion"
-  output_stream: "TRACKING:tracking_data"
-
-  node_options: {
-    [type.googleapis.com/mediapipe.FlowPackagerCalculatorOptions]: {
-      flow_packager_options: {
-        binary_tracking_data_support: false
-      }
-    }
-  }
-}
-
-# Tracks box positions over time.
-node: {
-  calculator: "BoxTrackerCalculator"
-  input_stream: "TRACKING:tracking_data"
-  input_stream: "TRACK_TIME:input_video"
-  input_stream: "START_POS:start_pos"
-  input_stream: "CANCEL_OBJECT_ID:cancel_object_id"
-  input_stream_info: {
-    tag_index: "CANCEL_OBJECT_ID"
-    back_edge: true
-  }
-  output_stream: "BOXES:boxes"
-
-  input_stream_handler {
-    input_stream_handler: "SyncSetInputStreamHandler"
-    options {
-      [mediapipe.SyncSetInputStreamHandlerOptions.ext] {
-        sync_set {
-          tag_index: "TRACKING"
-          tag_index: "TRACK_TIME"
-        }
-        sync_set {
-          tag_index: "START_POS"
-        }
-        sync_set {
-          tag_index: "CANCEL_OBJECT_ID"
-        }
-      }
-    }
-  }
-
-  node_options: {
-    [type.googleapis.com/mediapipe.BoxTrackerCalculatorOptions]: {
-      tracker_options: {
-        track_step_options {
-          track_object_and_camera: true
-          tracking_degrees: TRACKING_DEGREE_OBJECT_ROTATION_SCALE
-          inlier_spring_force: 0.0
-          static_motion_temporal_ratio: 3e-2
-        }
-      }
-      visualize_tracking_data: false
-      streaming_track_data_cache_size: 100
-    }
-  }
-}
-
-# Consolidates tracking and detection results.
-node {
-  calculator: "FrameAnnotationTrackerCalculator"
-  input_stream: "FRAME_ANNOTATION:objects"
-  input_stream: "TRACKED_BOXES:boxes"
-  output_stream: "TRACKED_FRAME_ANNOTATION:tracked_objects"
-  output_stream: "CANCEL_OBJECT_ID:cancel_object_id"
-  node_options: {
-    [type.googleapis.com/mediapipe.FrameAnnotationTrackerCalculatorOptions] {
-      img_width: 240
-      img_height: 320
-      iou_threshold: 0.1
-    }
-  }
-
-  input_stream_handler {
-    input_stream_handler: "SyncSetInputStreamHandler"
-    options {
-      [mediapipe.SyncSetInputStreamHandlerOptions.ext] {
-        sync_set {
-          tag_index: "FRAME_ANNOTATION"
-        }
-        sync_set {
-          tag_index: "TRACKED_BOXES"
-        }
-      }
-    }
-  }
-}
-
-# Lift the tracked 2D keypoints to 3D using EPnP algorithm.
-node {
-  calculator: "Lift2DFrameAnnotationTo3DCalculator"
-  input_stream: "FRAME_ANNOTATION:tracked_objects"
-  output_stream: "LIFTED_FRAME_ANNOTATION:lifted_tracked_objects"
-}

mediapipe/graphs/object_detection_3d/subgraphs/renderer_cpu.pbtxt

@@ -0,0 +1,57 @@
+# MediaPipe Objectron vertices/landmarks rendering CPU subgraph.
+
+type: "RendererSubgraph"
+
+input_stream: "IMAGE:input_image"
+input_stream: "LANDMARKS:landmarks"
+input_stream: "NORM_RECT:rect"
+output_stream: "IMAGE:output_image"
+
+# Converts landmarks to drawing primitives for annotation overlay.
+node {
+  calculator: "LandmarksToRenderDataCalculator"
+  input_stream: "NORM_LANDMARKS:landmarks"
+  output_stream: "RENDER_DATA:landmark_render_data"
+  node_options: {
+    [type.googleapis.com/mediapipe.LandmarksToRenderDataCalculatorOptions] {
+      landmark_connections: [1, 2] # edge 1-2
+      landmark_connections: [1, 3] # edge 1-3
+      landmark_connections: [1, 5] # edge 1-5
+      landmark_connections: [2, 4] # edge 2-4
+      landmark_connections: [2, 6] # edge 2-6
+      landmark_connections: [3, 4] # edge 3-4
+      landmark_connections: [3, 7] # edge 3-7
+      landmark_connections: [4, 8] # edge 4-8
+      landmark_connections: [5, 6] # edge 5-6
+      landmark_connections: [5, 7] # edge 5-7
+      landmark_connections: [6, 8] # edge 6-8
+      landmark_connections: [7, 8] # edge 7-8
+      landmark_color { r: 255 g: 0 b: 0 }
+      connection_color { r: 0 g: 255 b: 0 }
+      thickness: 4.0
+    }
+  }
+}
+
+# Converts normalized rects to drawing primitives for annotation overlay.
+node {
+  calculator: "RectToRenderDataCalculator"
+  input_stream: "NORM_RECT:rect"
+  output_stream: "RENDER_DATA:rect_render_data"
+  node_options: {
+    [type.googleapis.com/mediapipe.RectToRenderDataCalculatorOptions] {
+      filled: false
+      color { r: 255 g: 0 b: 0 }
+      thickness: 4.0
+    }
+  }
+}
+
+# Draws annotations and overlays them on top of the input images.
+node {
+  calculator: "AnnotationOverlayCalculator"
+  input_stream: "IMAGE:input_image"
+  input_stream: "landmark_render_data"
+  input_stream: "rect_render_data"
+  output_stream: "IMAGE:output_image"
+}

mediapipe/graphs/pose_tracking/BUILD

@@ -21,6 +21,42 @@ licenses(["notice"])
 
 package(default_visibility = ["//visibility:public"])
 
+cc_library(
+    name = "pose_tracking_gpu_deps",
+    deps = [
+        "//mediapipe/calculators/core:flow_limiter_calculator",
+        "//mediapipe/calculators/image:image_properties_calculator",
+        "//mediapipe/calculators/util:landmarks_smoothing_calculator",
+        "//mediapipe/graphs/pose_tracking/subgraphs:pose_renderer_gpu",
+        "//mediapipe/modules/pose_landmark:pose_landmark_gpu",
+    ],
+)
+
+mediapipe_binary_graph(
+    name = "pose_tracking_gpu_binary_graph",
+    graph = "pose_tracking_gpu.pbtxt",
+    output_name = "pose_tracking_gpu.binarypb",
+    deps = [":pose_tracking_gpu_deps"],
+)
+
+cc_library(
+    name = "pose_tracking_cpu_deps",
+    deps = [
+        "//mediapipe/calculators/core:flow_limiter_calculator",
+        "//mediapipe/calculators/image:image_properties_calculator",
+        "//mediapipe/calculators/util:landmarks_smoothing_calculator",
+        "//mediapipe/graphs/pose_tracking/subgraphs:pose_renderer_cpu",
+        "//mediapipe/modules/pose_landmark:pose_landmark_cpu",
+    ],
+)
+
+mediapipe_binary_graph(
+    name = "pose_tracking_cpu_binary_graph",
+    graph = "pose_tracking_cpu.pbtxt",
+    output_name = "pose_tracking_cpu.binarypb",
+    deps = [":pose_tracking_cpu_deps"],
+)
+
 cc_library(
     name = "upper_body_pose_tracking_gpu_deps",
     deps = [

mediapipe/graphs/pose_tracking/pose_tracking_cpu.pbtxt

@@ -0,0 +1,72 @@
+# MediaPipe graph that performs pose tracking with TensorFlow Lite on CPU.
+
+# CPU buffer. (ImageFrame)
+input_stream: "input_video"
+
+# Output image with rendered results. (ImageFrame)
+output_stream: "output_video"
+# Pose landmarks. (NormalizedLandmarkList)
+output_stream: "pose_landmarks"
+
+# Throttles the images flowing downstream for flow control. It passes through
+# the very first incoming image unaltered, and waits for downstream nodes
+# (calculators and subgraphs) in the graph to finish their tasks before it
+# passes through another image. All images that come in while waiting are
+# dropped, limiting the number of in-flight images in most part of the graph to
+# 1. This prevents the downstream nodes from queuing up incoming images and data
+# excessively, which leads to increased latency and memory usage, unwanted in
+# real-time mobile applications. It also eliminates unnecessarily computation,
+# e.g., the output produced by a node may get dropped downstream if the
+# subsequent nodes are still busy processing previous inputs.
+node {
+  calculator: "FlowLimiterCalculator"
+  input_stream: "input_video"
+  input_stream: "FINISHED:output_video"
+  input_stream_info: {
+    tag_index: "FINISHED"
+    back_edge: true
+  }
+  output_stream: "throttled_input_video"
+}
+
+# Subgraph that detects poses and corresponding landmarks.
+node {
+  calculator: "PoseLandmarkCpu"
+  input_stream: "IMAGE:throttled_input_video"
+  output_stream: "LANDMARKS:pose_landmarks"
+  output_stream: "DETECTION:pose_detection"
+  output_stream: "ROI_FROM_LANDMARKS:roi_from_landmarks"
+}
+
+# Calculates size of the image.
+node {
+  calculator: "ImagePropertiesCalculator"
+  input_stream: "IMAGE:throttled_input_video"
+  output_stream: "SIZE:image_size"
+}
+
+# Smoothes pose landmarks in order to reduce jitter.
+node {
+  calculator: "LandmarksSmoothingCalculator"
+  input_stream: "NORM_LANDMARKS:pose_landmarks"
+  input_stream: "IMAGE_SIZE:image_size"
+  output_stream: "NORM_FILTERED_LANDMARKS:pose_landmarks_smoothed"
+  node_options: {
+    [type.googleapis.com/mediapipe.LandmarksSmoothingCalculatorOptions] {
+      velocity_filter: {
+        window_size: 5
+        velocity_scale: 10.0
+      }
+    }
+  }
+}
+
+# Subgraph that renders pose-landmark annotation onto the input image.
+node {
+  calculator: "PoseRendererCpu"
+  input_stream: "IMAGE:throttled_input_video"
+  input_stream: "LANDMARKS:pose_landmarks_smoothed"
+  input_stream: "ROI:roi_from_landmarks"
+  input_stream: "DETECTION:pose_detection"
+  output_stream: "IMAGE:output_video"
+}

mediapipe/graphs/pose_tracking/pose_tracking_gpu.pbtxt

@@ -0,0 +1,72 @@
+# MediaPipe graph that performs pose tracking with TensorFlow Lite on GPU.
+
+# GPU buffer. (GpuBuffer)
+input_stream: "input_video"
+
+# Output image with rendered results. (GpuBuffer)
+output_stream: "output_video"
+# Pose landmarks. (NormalizedLandmarkList)
+output_stream: "pose_landmarks"
+
+# Throttles the images flowing downstream for flow control. It passes through
+# the very first incoming image unaltered, and waits for downstream nodes
+# (calculators and subgraphs) in the graph to finish their tasks before it
+# passes through another image. All images that come in while waiting are
+# dropped, limiting the number of in-flight images in most part of the graph to
+# 1. This prevents the downstream nodes from queuing up incoming images and data
+# excessively, which leads to increased latency and memory usage, unwanted in
+# real-time mobile applications. It also eliminates unnecessarily computation,
+# e.g., the output produced by a node may get dropped downstream if the
+# subsequent nodes are still busy processing previous inputs.
+node {
+  calculator: "FlowLimiterCalculator"
+  input_stream: "input_video"
+  input_stream: "FINISHED:output_video"
+  input_stream_info: {
+    tag_index: "FINISHED"
+    back_edge: true
+  }
+  output_stream: "throttled_input_video"
+}
+
+# Subgraph that detects poses and corresponding landmarks.
+node {
+  calculator: "PoseLandmarkGpu"
+  input_stream: "IMAGE:throttled_input_video"
+  output_stream: "LANDMARKS:pose_landmarks"
+  output_stream: "DETECTION:pose_detection"
+  output_stream: "ROI_FROM_LANDMARKS:roi_from_landmarks"
+}
+
+# Calculates size of the image.
+node {
+  calculator: "ImagePropertiesCalculator"
+  input_stream: "IMAGE_GPU:throttled_input_video"
+  output_stream: "SIZE:image_size"
+}
+
+# Smoothes pose landmarks in order to reduce jitter.
+node {
+  calculator: "LandmarksSmoothingCalculator"
+  input_stream: "NORM_LANDMARKS:pose_landmarks"
+  input_stream: "IMAGE_SIZE:image_size"
+  output_stream: "NORM_FILTERED_LANDMARKS:pose_landmarks_smoothed"
+  node_options: {
+    [type.googleapis.com/mediapipe.LandmarksSmoothingCalculatorOptions] {
+      velocity_filter: {
+        window_size: 5
+        velocity_scale: 10.0
+      }
+    }
+  }
+}
+
+# Subgraph that renders pose-landmark annotation onto the input image.
+node {
+  calculator: "PoseRendererGpu"
+  input_stream: "IMAGE:throttled_input_video"
+  input_stream: "LANDMARKS:pose_landmarks_smoothed"
+  input_stream: "ROI:roi_from_landmarks"
+  input_stream: "DETECTION:pose_detection"
+  output_stream: "IMAGE:output_video"
+}

mediapipe/graphs/pose_tracking/subgraphs/BUILD

@@ -21,6 +21,34 @@ licenses(["notice"])
 
 package(default_visibility = ["//visibility:public"])
 
+mediapipe_simple_subgraph(
+    name = "pose_renderer_gpu",
+    graph = "pose_renderer_gpu.pbtxt",
+    register_as = "PoseRendererGpu",
+    deps = [
+        "//mediapipe/calculators/core:split_normalized_landmark_list_calculator",
+        "//mediapipe/calculators/util:annotation_overlay_calculator",
+        "//mediapipe/calculators/util:detections_to_render_data_calculator",
+        "//mediapipe/calculators/util:landmarks_to_render_data_calculator",
+        "//mediapipe/calculators/util:rect_to_render_data_calculator",
+        "//mediapipe/calculators/util:rect_to_render_scale_calculator",
+    ],
+)
+
+mediapipe_simple_subgraph(
+    name = "pose_renderer_cpu",
+    graph = "pose_renderer_cpu.pbtxt",
+    register_as = "PoseRendererCpu",
+    deps = [
+        "//mediapipe/calculators/core:split_normalized_landmark_list_calculator",
+        "//mediapipe/calculators/util:annotation_overlay_calculator",
+        "//mediapipe/calculators/util:detections_to_render_data_calculator",
+        "//mediapipe/calculators/util:landmarks_to_render_data_calculator",
+        "//mediapipe/calculators/util:rect_to_render_data_calculator",
+        "//mediapipe/calculators/util:rect_to_render_scale_calculator",
+    ],
+)
+
 mediapipe_simple_subgraph(
     name = "upper_body_pose_renderer_gpu",
     graph = "upper_body_pose_renderer_gpu.pbtxt",

mediapipe/graphs/pose_tracking/subgraphs/pose_renderer_cpu.pbtxt

@@ -0,0 +1,274 @@
+# MediaPipe pose landmarks rendering subgraph.
+
+type: "PoseRendererCpu"
+
+# CPU image. (ImageFrame)
+input_stream: "IMAGE:input_image"
+# Pose landmarks. (NormalizedLandmarkList)
+input_stream: "LANDMARKS:pose_landmarks"
+# Region of interest calculated based on landmarks. (NormalizedRect)
+input_stream: "ROI:roi"
+# Detected pose. (Detection)
+input_stream: "DETECTION:detection"
+
+# CPU image with rendered data. (ImageFrame)
+output_stream: "IMAGE:output_image"
+
+node {
+  calculator: "ImagePropertiesCalculator"
+  input_stream: "IMAGE:input_image"
+  output_stream: "SIZE:image_size"
+}
+
+# Calculates rendering scale based on the pose roi.
+node {
+  calculator: "RectToRenderScaleCalculator"
+  input_stream: "NORM_RECT:roi"
+  input_stream: "IMAGE_SIZE:image_size"
+  output_stream: "RENDER_SCALE:render_scale"
+  node_options: {
+    [type.googleapis.com/mediapipe.RectToRenderScaleCalculatorOptions] {
+      multiplier: 0.0012
+    }
+  }
+}
+
+# Converts detections to drawing primitives for annotation overlay.
+node {
+  calculator: "DetectionsToRenderDataCalculator"
+  input_stream: "DETECTION:detection"
+  output_stream: "RENDER_DATA:detection_render_data"
+  node_options: {
+    [type.googleapis.com/mediapipe.DetectionsToRenderDataCalculatorOptions] {
+      thickness: 4.0
+      color { r: 0 g: 255 b: 0 }
+    }
+  }
+}
+
+node {
+  calculator: "SplitNormalizedLandmarkListCalculator"
+  input_stream: "pose_landmarks"
+  output_stream: "visible_pose_landmarks"
+  node_options: {
+    [type.googleapis.com/mediapipe.SplitVectorCalculatorOptions] {
+      ranges: { begin: 0 end: 25 }
+    }
+  }
+}
+
+# Converts landmarks to drawing primitives for annotation overlay.
+node {
+  calculator: "LandmarksToRenderDataCalculator"
+  input_stream: "NORM_LANDMARKS:pose_landmarks"
+  input_stream: "RENDER_SCALE:render_scale"
+  output_stream: "RENDER_DATA:landmarks_render_data"
+  node_options: {
+    [type.googleapis.com/mediapipe.LandmarksToRenderDataCalculatorOptions] {
+      landmark_connections: 0
+      landmark_connections: 1
+      landmark_connections: 1
+      landmark_connections: 2
+      landmark_connections: 2
+      landmark_connections: 3
+      landmark_connections: 3
+      landmark_connections: 7
+      landmark_connections: 0
+      landmark_connections: 4
+      landmark_connections: 4
+      landmark_connections: 5
+      landmark_connections: 5
+      landmark_connections: 6
+      landmark_connections: 6
+      landmark_connections: 8
+      landmark_connections: 9
+      landmark_connections: 10
+      landmark_connections: 11
+      landmark_connections: 12
+      landmark_connections: 11
+      landmark_connections: 13
+      landmark_connections: 13
+      landmark_connections: 15
+      landmark_connections: 15
+      landmark_connections: 17
+      landmark_connections: 15
+      landmark_connections: 19
+      landmark_connections: 15
+      landmark_connections: 21
+      landmark_connections: 17
+      landmark_connections: 19
+      landmark_connections: 12
+      landmark_connections: 14
+      landmark_connections: 14
+      landmark_connections: 16
+      landmark_connections: 16
+      landmark_connections: 18
+      landmark_connections: 16
+      landmark_connections: 20
+      landmark_connections: 16
+      landmark_connections: 22
+      landmark_connections: 18
+      landmark_connections: 20
+      landmark_connections: 11
+      landmark_connections: 23
+      landmark_connections: 12
+      landmark_connections: 24
+      landmark_connections: 23
+      landmark_connections: 24
+      landmark_connections: 23
+      landmark_connections: 25
+      landmark_connections: 24
+      landmark_connections: 26
+      landmark_connections: 25
+      landmark_connections: 27
+      landmark_connections: 26
+      landmark_connections: 28
+      landmark_connections: 27
+      landmark_connections: 29
+      landmark_connections: 28
+      landmark_connections: 30
+      landmark_connections: 29
+      landmark_connections: 31
+      landmark_connections: 30
+      landmark_connections: 32
+      landmark_connections: 27
+      landmark_connections: 31
+      landmark_connections: 28
+      landmark_connections: 32
+
+      landmark_color { r: 255 g: 255 b: 255 }
+      connection_color { r: 255 g: 255 b: 255 }
+      thickness: 3.0
+      visualize_landmark_depth: false
+      utilize_visibility: true
+      visibility_threshold: 0.5
+    }
+  }
+}
+
+# Take left pose landmarks.
+node {
+  calculator: "SplitNormalizedLandmarkListCalculator"
+  input_stream: "pose_landmarks"
+  output_stream: "landmarks_left_side"
+  node_options: {
+    [type.googleapis.com/mediapipe.SplitVectorCalculatorOptions] {
+      ranges: { begin: 1 end: 4 }
+      ranges: { begin: 7 end: 8 }
+      ranges: { begin: 9 end: 10 }
+      ranges: { begin: 11 end: 12 }
+      ranges: { begin: 13 end: 14 }
+      ranges: { begin: 15 end: 16 }
+      ranges: { begin: 17 end: 18 }
+      ranges: { begin: 19 end: 20 }
+      ranges: { begin: 21 end: 22 }
+      ranges: { begin: 23 end: 24 }
+
+      combine_outputs: true
+    }
+  }
+}
+
+# Take right pose landmarks.
+node {
+  calculator: "SplitNormalizedLandmarkListCalculator"
+  input_stream: "pose_landmarks"
+  output_stream: "landmarks_right_side"
+  node_options: {
+    [type.googleapis.com/mediapipe.SplitVectorCalculatorOptions] {
+      ranges: { begin: 4 end: 7 }
+      ranges: { begin: 8 end: 9 }
+      ranges: { begin: 10 end: 11 }
+      ranges: { begin: 12 end: 13 }
+      ranges: { begin: 14 end: 15 }
+      ranges: { begin: 16 end: 17 }
+      ranges: { begin: 18 end: 19 }
+      ranges: { begin: 20 end: 21 }
+      ranges: { begin: 22 end: 23 }
+      ranges: { begin: 24 end: 25 }
+
+      combine_outputs: true
+    }
+  }
+}
+
+# Render pose joints as big white circles.
+node {
+  calculator: "LandmarksToRenderDataCalculator"
+  input_stream: "NORM_LANDMARKS:visible_pose_landmarks"
+  input_stream: "RENDER_SCALE:render_scale"
+  output_stream: "RENDER_DATA:landmarks_background_joints_render_data"
+  node_options: {
+    [type.googleapis.com/mediapipe.LandmarksToRenderDataCalculatorOptions] {
+      landmark_color { r: 255 g: 255 b: 255 }
+      connection_color { r: 255 g: 255 b: 255 }
+      thickness: 5.0
+      visualize_landmark_depth: false
+      utilize_visibility: true
+      visibility_threshold: 0.5
+    }
+  }
+}
+
+# Render pose left side joints as orange circles (inside white ones).
+node {
+  calculator: "LandmarksToRenderDataCalculator"
+  input_stream: "NORM_LANDMARKS:landmarks_left_side"
+  input_stream: "RENDER_SCALE:render_scale"
+  output_stream: "RENDER_DATA:landmarks_left_joints_render_data"
+  node_options: {
+    [type.googleapis.com/mediapipe.LandmarksToRenderDataCalculatorOptions] {
+      landmark_color { r: 255 g: 138 b: 0 }
+      connection_color { r: 255 g: 138 b: 0 }
+      thickness: 3.0
+      visualize_landmark_depth: false
+      utilize_visibility: true
+      visibility_threshold: 0.5
+    }
+  }
+}
+
+# Render pose right side joints as cyan circles (inside white ones).
+node {
+  calculator: "LandmarksToRenderDataCalculator"
+  input_stream: "NORM_LANDMARKS:landmarks_right_side"
+  input_stream: "RENDER_SCALE:render_scale"
+  output_stream: "RENDER_DATA:landmarks_right_joints_render_data"
+  node_options: {
+    [type.googleapis.com/mediapipe.LandmarksToRenderDataCalculatorOptions] {
+      landmark_color { r: 0 g: 217 b: 231 }
+      connection_color { r: 0 g: 217 b: 231 }
+      thickness: 3.0
+      visualize_landmark_depth: false
+      utilize_visibility: true
+      visibility_threshold: 0.5
+    }
+  }
+}
+
+# Converts normalized rects to drawing primitives for annotation overlay.
+node {
+  calculator: "RectToRenderDataCalculator"
+  input_stream: "NORM_RECT:roi"
+  output_stream: "RENDER_DATA:roi_render_data"
+  node_options: {
+    [type.googleapis.com/mediapipe.RectToRenderDataCalculatorOptions] {
+      filled: false
+      color { r: 255 g: 0 b: 0 }
+      thickness: 4.0
+    }
+  }
+}
+
+# Draws annotations and overlays them on top of the input images.
+node {
+  calculator: "AnnotationOverlayCalculator"
+  input_stream: "IMAGE:input_image"
+  input_stream: "detection_render_data"
+  input_stream: "landmarks_render_data"
+  input_stream: "landmarks_background_joints_render_data"
+  input_stream: "landmarks_left_joints_render_data"
+  input_stream: "landmarks_right_joints_render_data"
+  input_stream: "roi_render_data"
+  output_stream: "IMAGE:output_image"
+}

mediapipe/graphs/pose_tracking/subgraphs/pose_renderer_gpu.pbtxt

@@ -0,0 +1,274 @@
+# MediaPipe pose landmarks rendering subgraph.
+
+type: "PoseRendererGpu"
+
+# GPU image. (GpuBuffer)
+input_stream: "IMAGE:input_image"
+# Pose landmarks. (NormalizedLandmarkList)
+input_stream: "LANDMARKS:pose_landmarks"
+# Region of interest calculated based on landmarks. (NormalizedRect)
+input_stream: "ROI:roi"
+# Detected pose. (Detection)
+input_stream: "DETECTION:detection"
+
+# GPU image with rendered data. (GpuBuffer)
+output_stream: "IMAGE:output_image"
+
+node {
+  calculator: "ImagePropertiesCalculator"
+  input_stream: "IMAGE_GPU:input_image"
+  output_stream: "SIZE:image_size"
+}
+
+# Calculates rendering scale based on the pose roi.
+node {
+  calculator: "RectToRenderScaleCalculator"
+  input_stream: "NORM_RECT:roi"
+  input_stream: "IMAGE_SIZE:image_size"
+  output_stream: "RENDER_SCALE:render_scale"
+  node_options: {
+    [type.googleapis.com/mediapipe.RectToRenderScaleCalculatorOptions] {
+      multiplier: 0.0012
+    }
+  }
+}
+
+# Converts detections to drawing primitives for annotation overlay.
+node {
+  calculator: "DetectionsToRenderDataCalculator"
+  input_stream: "DETECTION:detection"
+  output_stream: "RENDER_DATA:detection_render_data"
+  node_options: {
+    [type.googleapis.com/mediapipe.DetectionsToRenderDataCalculatorOptions] {
+      thickness: 4.0
+      color { r: 0 g: 255 b: 0 }
+    }
+  }
+}
+
+node {
+  calculator: "SplitNormalizedLandmarkListCalculator"
+  input_stream: "pose_landmarks"
+  output_stream: "visible_pose_landmarks"
+  node_options: {
+    [type.googleapis.com/mediapipe.SplitVectorCalculatorOptions] {
+      ranges: { begin: 0 end: 25 }
+    }
+  }
+}
+
+# Converts landmarks to drawing primitives for annotation overlay.
+node {
+  calculator: "LandmarksToRenderDataCalculator"
+  input_stream: "NORM_LANDMARKS:pose_landmarks"
+  input_stream: "RENDER_SCALE:render_scale"
+  output_stream: "RENDER_DATA:landmarks_render_data"
+  node_options: {
+    [type.googleapis.com/mediapipe.LandmarksToRenderDataCalculatorOptions] {
+      landmark_connections: 0
+      landmark_connections: 1
+      landmark_connections: 1
+      landmark_connections: 2
+      landmark_connections: 2
+      landmark_connections: 3
+      landmark_connections: 3
+      landmark_connections: 7
+      landmark_connections: 0
+      landmark_connections: 4
+      landmark_connections: 4
+      landmark_connections: 5
+      landmark_connections: 5
+      landmark_connections: 6
+      landmark_connections: 6
+      landmark_connections: 8
+      landmark_connections: 9
+      landmark_connections: 10
+      landmark_connections: 11
+      landmark_connections: 12
+      landmark_connections: 11
+      landmark_connections: 13
+      landmark_connections: 13
+      landmark_connections: 15
+      landmark_connections: 15
+      landmark_connections: 17
+      landmark_connections: 15
+      landmark_connections: 19
+      landmark_connections: 15
+      landmark_connections: 21
+      landmark_connections: 17
+      landmark_connections: 19
+      landmark_connections: 12
+      landmark_connections: 14
+      landmark_connections: 14
+      landmark_connections: 16
+      landmark_connections: 16
+      landmark_connections: 18
+      landmark_connections: 16
+      landmark_connections: 20
+      landmark_connections: 16
+      landmark_connections: 22
+      landmark_connections: 18
+      landmark_connections: 20
+      landmark_connections: 11
+      landmark_connections: 23
+      landmark_connections: 12
+      landmark_connections: 24
+      landmark_connections: 23
+      landmark_connections: 24
+      landmark_connections: 23
+      landmark_connections: 25
+      landmark_connections: 24
+      landmark_connections: 26
+      landmark_connections: 25
+      landmark_connections: 27
+      landmark_connections: 26
+      landmark_connections: 28
+      landmark_connections: 27
+      landmark_connections: 29
+      landmark_connections: 28
+      landmark_connections: 30
+      landmark_connections: 29
+      landmark_connections: 31
+      landmark_connections: 30
+      landmark_connections: 32
+      landmark_connections: 27
+      landmark_connections: 31
+      landmark_connections: 28
+      landmark_connections: 32
+
+      landmark_color { r: 255 g: 255 b: 255 }
+      connection_color { r: 255 g: 255 b: 255 }
+      thickness: 3.0
+      visualize_landmark_depth: false
+      utilize_visibility: true
+      visibility_threshold: 0.5
+    }
+  }
+}
+
+# Take left pose landmarks.
+node {
+  calculator: "SplitNormalizedLandmarkListCalculator"
+  input_stream: "pose_landmarks"
+  output_stream: "landmarks_left_side"
+  node_options: {
+    [type.googleapis.com/mediapipe.SplitVectorCalculatorOptions] {
+      ranges: { begin: 1 end: 4 }
+      ranges: { begin: 7 end: 8 }
+      ranges: { begin: 9 end: 10 }
+      ranges: { begin: 11 end: 12 }
+      ranges: { begin: 13 end: 14 }
+      ranges: { begin: 15 end: 16 }
+      ranges: { begin: 17 end: 18 }
+      ranges: { begin: 19 end: 20 }
+      ranges: { begin: 21 end: 22 }
+      ranges: { begin: 23 end: 24 }
+
+      combine_outputs: true
+    }
+  }
+}
+
+# Take right pose landmarks.
+node {
+  calculator: "SplitNormalizedLandmarkListCalculator"
+  input_stream: "pose_landmarks"
+  output_stream: "landmarks_right_side"
+  node_options: {
+    [type.googleapis.com/mediapipe.SplitVectorCalculatorOptions] {
+      ranges: { begin: 4 end: 7 }
+      ranges: { begin: 8 end: 9 }
+      ranges: { begin: 10 end: 11 }
+      ranges: { begin: 12 end: 13 }
+      ranges: { begin: 14 end: 15 }
+      ranges: { begin: 16 end: 17 }
+      ranges: { begin: 18 end: 19 }
+      ranges: { begin: 20 end: 21 }
+      ranges: { begin: 22 end: 23 }
+      ranges: { begin: 24 end: 25 }
+
+      combine_outputs: true
+    }
+  }
+}
+
+# Render pose joints as big white circles.
+node {
+  calculator: "LandmarksToRenderDataCalculator"
+  input_stream: "NORM_LANDMARKS:visible_pose_landmarks"
+  input_stream: "RENDER_SCALE:render_scale"
+  output_stream: "RENDER_DATA:landmarks_background_joints_render_data"
+  node_options: {
+    [type.googleapis.com/mediapipe.LandmarksToRenderDataCalculatorOptions] {
+      landmark_color { r: 255 g: 255 b: 255 }
+      connection_color { r: 255 g: 255 b: 255 }
+      thickness: 5.0
+      visualize_landmark_depth: false
+      utilize_visibility: true
+      visibility_threshold: 0.5
+    }
+  }
+}
+
+# Render pose left side joints as orange circles (inside white ones).
+node {
+  calculator: "LandmarksToRenderDataCalculator"
+  input_stream: "NORM_LANDMARKS:landmarks_left_side"
+  input_stream: "RENDER_SCALE:render_scale"
+  output_stream: "RENDER_DATA:landmarks_left_joints_render_data"
+  node_options: {
+    [type.googleapis.com/mediapipe.LandmarksToRenderDataCalculatorOptions] {
+      landmark_color { r: 255 g: 138 b: 0 }
+      connection_color { r: 255 g: 138 b: 0 }
+      thickness: 3.0
+      visualize_landmark_depth: false
+      utilize_visibility: true
+      visibility_threshold: 0.5
+    }
+  }
+}
+
+# Render pose right side joints as cyan circles (inside white ones).
+node {
+  calculator: "LandmarksToRenderDataCalculator"
+  input_stream: "NORM_LANDMARKS:landmarks_right_side"
+  input_stream: "RENDER_SCALE:render_scale"
+  output_stream: "RENDER_DATA:landmarks_right_joints_render_data"
+  node_options: {
+    [type.googleapis.com/mediapipe.LandmarksToRenderDataCalculatorOptions] {
+      landmark_color { r: 0 g: 217 b: 231 }
+      connection_color { r: 0 g: 217 b: 231 }
+      thickness: 3.0
+      visualize_landmark_depth: false
+      utilize_visibility: true
+      visibility_threshold: 0.5
+    }
+  }
+}
+
+# Converts normalized rects to drawing primitives for annotation overlay.
+node {
+  calculator: "RectToRenderDataCalculator"
+  input_stream: "NORM_RECT:roi"
+  output_stream: "RENDER_DATA:roi_render_data"
+  node_options: {
+    [type.googleapis.com/mediapipe.RectToRenderDataCalculatorOptions] {
+      filled: false
+      color { r: 255 g: 0 b: 0 }
+      thickness: 4.0
+    }
+  }
+}
+
+# Draws annotations and overlays them on top of the input images.
+node {
+  calculator: "AnnotationOverlayCalculator"
+  input_stream: "IMAGE_GPU:input_image"
+  input_stream: "detection_render_data"
+  input_stream: "landmarks_render_data"
+  input_stream: "landmarks_background_joints_render_data"
+  input_stream: "landmarks_left_joints_render_data"
+  input_stream: "landmarks_right_joints_render_data"
+  input_stream: "roi_render_data"
+  output_stream: "IMAGE_GPU:output_image"
+}

mediapipe/graphs/pose_tracking/subgraphs/upper_body_pose_renderer_cpu.pbtxt

@@ -121,7 +121,7 @@ node {
       thickness: 3.0
       visualize_landmark_depth: false
       utilize_visibility: true
-      visibility_threshold: 0.1
+      visibility_threshold: 0.5
     }
   }
 }
@@ -185,7 +185,7 @@ node {
       thickness: 5.0
       visualize_landmark_depth: false
       utilize_visibility: true
-      visibility_threshold: 0.1
+      visibility_threshold: 0.5
     }
   }
 }
@@ -203,7 +203,7 @@ node {
       thickness: 3.0
       visualize_landmark_depth: false
       utilize_visibility: true
-      visibility_threshold: 0.1
+      visibility_threshold: 0.5
     }
   }
 }
@@ -221,7 +221,7 @@ node {
       thickness: 3.0
       visualize_landmark_depth: false
       utilize_visibility: true
-      visibility_threshold: 0.1
+      visibility_threshold: 0.5
     }
   }
 }

mediapipe/graphs/pose_tracking/subgraphs/upper_body_pose_renderer_gpu.pbtxt

@@ -121,7 +121,7 @@ node {
       thickness: 3.0
       visualize_landmark_depth: false
       utilize_visibility: true
-      visibility_threshold: 0.1
+      visibility_threshold: 0.5
     }
   }
 }
@@ -185,7 +185,7 @@ node {
       thickness: 5.0
       visualize_landmark_depth: false
       utilize_visibility: true
-      visibility_threshold: 0.1
+      visibility_threshold: 0.5
     }
   }
 }
@@ -203,7 +203,7 @@ node {
       thickness: 3.0
       visualize_landmark_depth: false
       utilize_visibility: true
-      visibility_threshold: 0.1
+      visibility_threshold: 0.5
     }
   }
 }
@@ -221,7 +221,7 @@ node {
       thickness: 3.0
       visualize_landmark_depth: false
       utilize_visibility: true
-      visibility_threshold: 0.1
+      visibility_threshold: 0.5
     }
   }
 }