Project import generated by Copybara.

GitOrigin-RevId: afeb9cf5a8c069c0a566d16e1622bbb086170e4d

mediapipe/graphs/edge_detection/edge_detection_mobile_gpu.pbtxt

@@ -1,6 +1,6 @@
 # MediaPipe graph that performs GPU Sobel edge detection on a live video stream.
 # Used in the examples in
-# mediapipe/examples/android/src/java/com/mediapipe/apps/edgedetectiongpu and
+# mediapipe/examples/android/src/java/com/mediapipe/apps/skeleton and
 # mediapipe/examples/ios/edgedetectiongpu.
 
 # Images coming into and out of the graph.

mediapipe/graphs/face_detection/face_detection_back_desktop_live.pbtxt

@@ -0,0 +1,184 @@
+# MediaPipe graph that performs face detection with TensorFlow Lite on CPU.
+# Used in the examples in
+# mediapipe/examples/desktop/face_detection:face_detection_cpu.
+
+# Images on GPU coming into and out of the graph.
+input_stream: "input_video"
+output_stream: "output_video"
+
+# Throttles the images flowing downstream for flow control. It passes through
+# the very first incoming image unaltered, and waits for
+# TfLiteTensorsToDetectionsCalculator downstream in the graph to finish
+# generating the corresponding detections before it passes through another
+# image. All images that come in while waiting are dropped, limiting the number
+# of in-flight images between this calculator and
+# TfLiteTensorsToDetectionsCalculator to 1. This prevents the nodes in between
+# from queuing up incoming images and data excessively, which leads to increased
+# latency and memory usage, unwanted in real-time mobile applications. It also
+# eliminates unnecessarily computation, e.g., a transformed image produced by
+# ImageTransformationCalculator may get dropped downstream if the subsequent
+# TfLiteConverterCalculator or TfLiteInferenceCalculator is still busy
+# processing previous inputs.
+node {
+  calculator: "FlowLimiterCalculator"
+  input_stream: "input_video"
+  input_stream: "FINISHED:detections"
+  input_stream_info: {
+    tag_index: "FINISHED"
+    back_edge: true
+  }
+  output_stream: "throttled_input_video"
+}
+
+# Transforms the input image on CPU to a 128x128 image. To scale the input
+# image, the scale_mode option is set to FIT to preserve the aspect ratio,
+# resulting in potential letterboxing in the transformed image.
+node: {
+  calculator: "ImageTransformationCalculator"
+  input_stream: "IMAGE:throttled_input_video"
+  output_stream: "IMAGE:transformed_input_video_cpu"
+  output_stream: "LETTERBOX_PADDING:letterbox_padding"
+  node_options: {
+    [type.googleapis.com/mediapipe.ImageTransformationCalculatorOptions] {
+      output_width: 256
+      output_height: 256
+      scale_mode: FIT
+    }
+  }
+}
+
+# Converts the transformed input image on CPU into an image tensor stored as a
+# TfLiteTensor.
+node {
+  calculator: "TfLiteConverterCalculator"
+  input_stream: "IMAGE:transformed_input_video_cpu"
+  output_stream: "TENSORS:image_tensor"
+}
+
+# Runs a TensorFlow Lite model on CPU that takes an image tensor and outputs a
+# vector of tensors representing, for instance, detection boxes/keypoints and
+# scores.
+node {
+  calculator: "TfLiteInferenceCalculator"
+  input_stream: "TENSORS:image_tensor"
+  output_stream: "TENSORS:detection_tensors"
+  node_options: {
+    [type.googleapis.com/mediapipe.TfLiteInferenceCalculatorOptions] {
+      model_path: "mediapipe/models/face_detection_back.tflite"
+    }
+  }
+}
+
+# Generates a single side packet containing a vector of SSD anchors based on
+# the specification in the options.
+node {
+  calculator: "SsdAnchorsCalculator"
+  output_side_packet: "anchors"
+  node_options: {
+    [type.googleapis.com/mediapipe.SsdAnchorsCalculatorOptions] {
+      num_layers: 4
+      min_scale: 0.15625
+      max_scale: 0.75
+      input_size_height: 256
+      input_size_width: 256
+      anchor_offset_x: 0.5
+      anchor_offset_y: 0.5
+      strides: 16
+      strides: 32
+      strides: 32
+      strides: 32
+      aspect_ratios: 1.0
+      fixed_anchor_size: true
+    }
+  }
+}
+
+# Decodes the detection tensors generated by the TensorFlow Lite model, based on
+# the SSD anchors and the specification in the options, into a vector of
+# detections. Each detection describes a detected object.
+node {
+  calculator: "TfLiteTensorsToDetectionsCalculator"
+  input_stream: "TENSORS:detection_tensors"
+  input_side_packet: "ANCHORS:anchors"
+  output_stream: "DETECTIONS:detections"
+  node_options: {
+    [type.googleapis.com/mediapipe.TfLiteTensorsToDetectionsCalculatorOptions] {
+      num_classes: 1
+      num_boxes: 896
+      num_coords: 16
+      box_coord_offset: 0
+      keypoint_coord_offset: 4
+      num_keypoints: 6
+      num_values_per_keypoint: 2
+      sigmoid_score: true
+      score_clipping_thresh: 100.0
+      reverse_output_order: true
+      x_scale: 256.0
+      y_scale: 256.0
+      h_scale: 256.0
+      w_scale: 256.0
+      min_score_thresh: 0.65
+    }
+  }
+}
+
+# Performs non-max suppression to remove excessive detections.
+node {
+  calculator: "NonMaxSuppressionCalculator"
+  input_stream: "detections"
+  output_stream: "filtered_detections"
+  node_options: {
+    [type.googleapis.com/mediapipe.NonMaxSuppressionCalculatorOptions] {
+      min_suppression_threshold: 0.3
+      overlap_type: INTERSECTION_OVER_UNION
+      algorithm: WEIGHTED
+      return_empty_detections: true
+    }
+  }
+}
+
+# Maps detection label IDs to the corresponding label text ("Face"). The label
+# map is provided in the label_map_path option.
+node {
+  calculator: "DetectionLabelIdToTextCalculator"
+  input_stream: "filtered_detections"
+  output_stream: "labeled_detections"
+  node_options: {
+    [type.googleapis.com/mediapipe.DetectionLabelIdToTextCalculatorOptions] {
+      label_map_path: "mediapipe/models/face_detection_back_labelmap.txt"
+    }
+  }
+}
+
+# Adjusts detection locations (already normalized to [0.f, 1.f]) on the
+# letterboxed image (after image transformation with the FIT scale mode) to the
+# corresponding locations on the same image with the letterbox removed (the
+# input image to the graph before image transformation).
+node {
+  calculator: "DetectionLetterboxRemovalCalculator"
+  input_stream: "DETECTIONS:labeled_detections"
+  input_stream: "LETTERBOX_PADDING:letterbox_padding"
+  output_stream: "DETECTIONS:output_detections"
+}
+
+# Converts the detections to drawing primitives for annotation overlay.
+node {
+  calculator: "DetectionsToRenderDataCalculator"
+  input_stream: "DETECTIONS:output_detections"
+  output_stream: "RENDER_DATA:render_data"
+  node_options: {
+    [type.googleapis.com/mediapipe.DetectionsToRenderDataCalculatorOptions] {
+      thickness: 4.0
+      color { r: 255 g: 0 b: 0 }
+    }
+  }
+}
+
+# Draws annotations and overlays them on top of the input images.
+node {
+  calculator: "AnnotationOverlayCalculator"
+  input_stream: "IMAGE:throttled_input_video"
+  input_stream: "render_data"
+  output_stream: "IMAGE:output_video"
+}
+

mediapipe/graphs/face_detection/face_detection_back_mobile_gpu.pbtxt

@@ -0,0 +1,184 @@
+# MediaPipe graph that performs face detection with TensorFlow Lite on GPU.
+# Used in the examples in
+# mediapipie/examples/android/src/java/com/mediapipe/apps/facedetectiongpu and
+# mediapipie/examples/ios/facedetectiongpu.
+
+# Images on GPU coming into and out of the graph.
+input_stream: "input_video"
+output_stream: "output_video"
+
+# Throttles the images flowing downstream for flow control. It passes through
+# the very first incoming image unaltered, and waits for
+# TfLiteTensorsToDetectionsCalculator downstream in the graph to finish
+# generating the corresponding detections before it passes through another
+# image. All images that come in while waiting are dropped, limiting the number
+# of in-flight images between this calculator and
+# TfLiteTensorsToDetectionsCalculator to 1. This prevents the nodes in between
+# from queuing up incoming images and data excessively, which leads to increased
+# latency and memory usage, unwanted in real-time mobile applications. It also
+# eliminates unnecessarily computation, e.g., a transformed image produced by
+# ImageTransformationCalculator may get dropped downstream if the subsequent
+# TfLiteConverterCalculator or TfLiteInferenceCalculator is still busy
+# processing previous inputs.
+node {
+  calculator: "FlowLimiterCalculator"
+  input_stream: "input_video"
+  input_stream: "FINISHED:detections"
+  input_stream_info: {
+    tag_index: "FINISHED"
+    back_edge: true
+  }
+  output_stream: "throttled_input_video"
+}
+
+# Transforms the input image on GPU to a 128x128 image. To scale the input
+# image, the scale_mode option is set to FIT to preserve the aspect ratio,
+# resulting in potential letterboxing in the transformed image.
+node: {
+  calculator: "ImageTransformationCalculator"
+  input_stream: "IMAGE_GPU:throttled_input_video"
+  output_stream: "IMAGE_GPU:transformed_input_video"
+  output_stream: "LETTERBOX_PADDING:letterbox_padding"
+  node_options: {
+    [type.googleapis.com/mediapipe.ImageTransformationCalculatorOptions] {
+      output_width: 256
+      output_height: 256
+      scale_mode: FIT
+    }
+  }
+}
+
+# Converts the transformed input image on GPU into an image tensor stored as a
+# TfLiteTensor.
+node {
+  calculator: "TfLiteConverterCalculator"
+  input_stream: "IMAGE_GPU:transformed_input_video"
+  output_stream: "TENSORS_GPU:image_tensor"
+}
+
+# Runs a TensorFlow Lite model on GPU that takes an image tensor and outputs a
+# vector of tensors representing, for instance, detection boxes/keypoints and
+# scores.
+node {
+  calculator: "TfLiteInferenceCalculator"
+  input_stream: "TENSORS_GPU:image_tensor"
+  output_stream: "TENSORS_GPU:detection_tensors"
+  node_options: {
+    [type.googleapis.com/mediapipe.TfLiteInferenceCalculatorOptions] {
+      model_path: "mediapipe/models/face_detection_back.tflite"
+    }
+  }
+}
+
+# Generates a single side packet containing a vector of SSD anchors based on
+# the specification in the options.
+node {
+  calculator: "SsdAnchorsCalculator"
+  output_side_packet: "anchors"
+  node_options: {
+    [type.googleapis.com/mediapipe.SsdAnchorsCalculatorOptions] {
+      num_layers: 4
+      min_scale: 0.15625
+      max_scale: 0.75
+      input_size_height: 256
+      input_size_width: 256
+      anchor_offset_x: 0.5
+      anchor_offset_y: 0.5
+      strides: 16
+      strides: 32
+      strides: 32
+      strides: 32
+      aspect_ratios: 1.0
+      fixed_anchor_size: true
+    }
+  }
+}
+
+# Decodes the detection tensors generated by the TensorFlow Lite model, based on
+# the SSD anchors and the specification in the options, into a vector of
+# detections. Each detection describes a detected object.
+node {
+  calculator: "TfLiteTensorsToDetectionsCalculator"
+  input_stream: "TENSORS_GPU:detection_tensors"
+  input_side_packet: "ANCHORS:anchors"
+  output_stream: "DETECTIONS:detections"
+  node_options: {
+    [type.googleapis.com/mediapipe.TfLiteTensorsToDetectionsCalculatorOptions] {
+      num_classes: 1
+      num_boxes: 896
+      num_coords: 16
+      box_coord_offset: 0
+      keypoint_coord_offset: 4
+      num_keypoints: 6
+      num_values_per_keypoint: 2
+      sigmoid_score: true
+      score_clipping_thresh: 100.0
+      reverse_output_order: true
+      x_scale: 256.0
+      y_scale: 256.0
+      h_scale: 256.0
+      w_scale: 256.0
+      min_score_thresh: 0.65
+    }
+  }
+}
+
+# Performs non-max suppression to remove excessive detections.
+node {
+  calculator: "NonMaxSuppressionCalculator"
+  input_stream: "detections"
+  output_stream: "filtered_detections"
+  node_options: {
+    [type.googleapis.com/mediapipe.NonMaxSuppressionCalculatorOptions] {
+      min_suppression_threshold: 0.3
+      overlap_type: INTERSECTION_OVER_UNION
+      algorithm: WEIGHTED
+      return_empty_detections: true
+    }
+  }
+}
+
+# Maps detection label IDs to the corresponding label text ("Face"). The label
+# map is provided in the label_map_path option.
+node {
+  calculator: "DetectionLabelIdToTextCalculator"
+  input_stream: "filtered_detections"
+  output_stream: "labeled_detections"
+  node_options: {
+    [type.googleapis.com/mediapipe.DetectionLabelIdToTextCalculatorOptions] {
+      label_map_path: "mediapipe/models/face_detection_back_labelmap.txt"
+    }
+  }
+}
+
+# Adjusts detection locations (already normalized to [0.f, 1.f]) on the
+# letterboxed image (after image transformation with the FIT scale mode) to the
+# corresponding locations on the same image with the letterbox removed (the
+# input image to the graph before image transformation).
+node {
+  calculator: "DetectionLetterboxRemovalCalculator"
+  input_stream: "DETECTIONS:labeled_detections"
+  input_stream: "LETTERBOX_PADDING:letterbox_padding"
+  output_stream: "DETECTIONS:output_detections"
+}
+
+# Converts the detections to drawing primitives for annotation overlay.
+node {
+  calculator: "DetectionsToRenderDataCalculator"
+  input_stream: "DETECTIONS:output_detections"
+  output_stream: "RENDER_DATA:render_data"
+  node_options: {
+    [type.googleapis.com/mediapipe.DetectionsToRenderDataCalculatorOptions] {
+      thickness: 4.0
+      color { r: 255 g: 0 b: 0 }
+    }
+  }
+}
+
+# Draws annotations and overlays them on top of the input images.
+node {
+  calculator: "AnnotationOverlayCalculator"
+  input_stream: "IMAGE_GPU:throttled_input_video"
+  input_stream: "render_data"
+  output_stream: "IMAGE_GPU:output_video"
+}

mediapipe/graphs/face_detection/face_detection_desktop_live.pbtxt

@@ -117,7 +117,7 @@ node {
       y_scale: 128.0
       h_scale: 128.0
       w_scale: 128.0
-      min_score_thresh: 0.75
+      min_score_thresh: 0.5
     }
   }
 }

mediapipe/graphs/face_detection/face_detection_mobile_cpu.pbtxt

@@ -128,7 +128,7 @@ node {
       y_scale: 128.0
       h_scale: 128.0
       w_scale: 128.0
-      min_score_thresh: 0.75
+      min_score_thresh: 0.5
     }
   }
 }

mediapipe/graphs/face_detection/face_detection_mobile_gpu.pbtxt

@@ -118,7 +118,7 @@ node {
       y_scale: 128.0
       h_scale: 128.0
       w_scale: 128.0
-      min_score_thresh: 0.75
+      min_score_thresh: 0.5
     }
   }
 }

mediapipe/graphs/hand_tracking/calculators/BUILD

@@ -0,0 +1,35 @@
+# Copyright 2020 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+licenses(["notice"])  # Apache 2.0
+
+package(default_visibility = ["//visibility:public"])
+
+cc_library(
+    name = "hand_detections_to_rects_calculator",
+    srcs = ["hand_detections_to_rects_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/calculators/util:detections_to_rects_calculator",
+        "//mediapipe/calculators/util:detections_to_rects_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:calculator_options_cc_proto",
+        "//mediapipe/framework/formats:detection_cc_proto",
+        "//mediapipe/framework/formats:location_data_cc_proto",
+        "//mediapipe/framework/formats:rect_cc_proto",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+    ],
+    alwayslink = 1,
+)

mediapipe/graphs/hand_tracking/calculators/hand_detections_to_rects_calculator.cc

@@ -0,0 +1,98 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <cmath>
+
+#include "mediapipe/calculators/util/detections_to_rects_calculator.h"
+#include "mediapipe/calculators/util/detections_to_rects_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/calculator_options.pb.h"
+#include "mediapipe/framework/formats/detection.pb.h"
+#include "mediapipe/framework/formats/location_data.pb.h"
+#include "mediapipe/framework/formats/rect.pb.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+
+namespace {
+
+// Indices of joints used for computing the rotation of the output rectangle
+// from detection box with keypoints.
+constexpr int kWristJoint = 0;
+constexpr int kMiddleFingerPIPJoint = 6;
+constexpr int kIndexFingerPIPJoint = 4;
+constexpr int kRingFingerPIPJoint = 8;
+constexpr char kImageSizeTag[] = "IMAGE_SIZE";
+
+}  // namespace
+
+// A calculator that converts Hand detection to a bounding box NormalizedRect.
+// The calculator overwrites the default logic of DetectionsToRectsCalculator
+// for rotating the detection bounding to a rectangle. The rotation angle is
+// computed based on 1) the wrist joint and 2) the average of PIP joints of
+// index finger, middle finger and ring finger. After rotation, the vector from
+// the wrist to the mean of PIP joints is expected to be vertical with wrist at
+// the bottom and the mean of PIP joints at the top.
+class HandDetectionsToRectsCalculator : public DetectionsToRectsCalculator {
+ public:
+  ::mediapipe::Status Open(CalculatorContext* cc) override {
+    RET_CHECK(cc->Inputs().HasTag(kImageSizeTag))
+        << "Image size is required to calculate rotated rect";
+    cc->SetOffset(TimestampDiff(0));
+    target_angle_ = M_PI * 0.5f;
+    rotate_ = true;
+    options_ = cc->Options<DetectionsToRectsCalculatorOptions>();
+    output_zero_rect_for_empty_detections_ =
+        options_.output_zero_rect_for_empty_detections();
+
+    return ::mediapipe::OkStatus();
+  }
+
+ private:
+  ::mediapipe::Status ComputeRotation(const ::mediapipe::Detection& detection,
+                                      const DetectionSpec& detection_spec,
+                                      float* rotation) override;
+};
+REGISTER_CALCULATOR(HandDetectionsToRectsCalculator);
+
+::mediapipe::Status HandDetectionsToRectsCalculator::ComputeRotation(
+    const Detection& detection, const DetectionSpec& detection_spec,
+    float* rotation) {
+  const auto& location_data = detection.location_data();
+  const auto& image_size = detection_spec.image_size;
+  RET_CHECK(image_size) << "Image size is required to calculate rotation";
+
+  const float x0 =
+      location_data.relative_keypoints(kWristJoint).x() * image_size->first;
+  const float y0 =
+      location_data.relative_keypoints(kWristJoint).y() * image_size->second;
+
+  float x1 = (location_data.relative_keypoints(kIndexFingerPIPJoint).x() +
+              location_data.relative_keypoints(kRingFingerPIPJoint).x()) /
+             2.f;
+  float y1 = (location_data.relative_keypoints(kIndexFingerPIPJoint).y() +
+              location_data.relative_keypoints(kRingFingerPIPJoint).y()) /
+             2.f;
+  x1 = (x1 + location_data.relative_keypoints(kMiddleFingerPIPJoint).x()) /
+       2.f * image_size->first;
+  y1 = (y1 + location_data.relative_keypoints(kMiddleFingerPIPJoint).y()) /
+       2.f * image_size->second;
+
+  *rotation = NormalizeRadians(target_angle_ - std::atan2(-(y1 - y0), x1 - x0));
+
+  return ::mediapipe::OkStatus();
+}
+
+}  // namespace mediapipe

mediapipe/graphs/hand_tracking/hand_tracking_desktop.pbtxt

@@ -64,6 +64,7 @@ node {
   input_stream: "NORM_RECT:hand_rect"
   output_stream: "LANDMARKS:hand_landmarks"
   output_stream: "NORM_RECT:hand_rect_from_landmarks"
+  output_stream: "HANDEDNESS:handedness"
   output_stream: "PRESENCE:hand_presence"
 }
 
@@ -107,6 +108,7 @@ node {
   input_stream: "LANDMARKS:hand_landmarks"
   input_stream: "NORM_RECT:hand_rect"
   input_stream: "DETECTIONS:palm_detections"
+  input_stream: "HANDEDNESS:handedness"
   output_stream: "IMAGE:output_video"
 }
 

mediapipe/graphs/hand_tracking/hand_tracking_desktop_live.pbtxt

@@ -55,6 +55,7 @@ node {
   input_stream: "NORM_RECT:hand_rect"
   output_stream: "LANDMARKS:hand_landmarks"
   output_stream: "NORM_RECT:hand_rect_from_landmarks"
+  output_stream: "HANDEDNESS:handedness"
   output_stream: "PRESENCE:hand_presence"
 }
 
@@ -98,6 +99,7 @@ node {
   input_stream: "LANDMARKS:hand_landmarks"
   input_stream: "NORM_RECT:hand_rect"
   input_stream: "DETECTIONS:palm_detections"
+  input_stream: "HANDEDNESS:handedness"
   output_stream: "IMAGE:output_video"
 }
 

mediapipe/graphs/hand_tracking/hand_tracking_mobile.pbtxt

@@ -77,6 +77,7 @@ node {
   output_stream: "LANDMARKS:hand_landmarks"
   output_stream: "NORM_RECT:hand_rect_from_landmarks"
   output_stream: "PRESENCE:hand_presence"
+  output_stream: "HANDEDNESS:handedness"
 }
 
 # Caches a hand rectangle fed back from HandLandmarkSubgraph, and upon the
@@ -119,5 +120,6 @@ node {
   input_stream: "LANDMARKS:hand_landmarks"
   input_stream: "NORM_RECT:hand_rect"
   input_stream: "DETECTIONS:palm_detections"
+  input_stream: "HANDEDNESS:handedness"
   output_stream: "IMAGE:output_video"
 }

mediapipe/graphs/hand_tracking/subgraphs/BUILD

@@ -70,12 +70,15 @@ mediapipe_simple_subgraph(
     graph = "hand_landmark_cpu.pbtxt",
     register_as = "HandLandmarkSubgraph",
     deps = [
+        "//mediapipe/calculators/core:split_normalized_landmark_list_calculator",
         "//mediapipe/calculators/core:split_vector_calculator",
         "//mediapipe/calculators/image:image_cropping_calculator",
         "//mediapipe/calculators/image:image_properties_calculator",
         "//mediapipe/calculators/image:image_transformation_calculator",
         "//mediapipe/calculators/tflite:tflite_converter_calculator",
+        "//mediapipe/calculators/tflite:tflite_custom_op_resolver_calculator",
         "//mediapipe/calculators/tflite:tflite_inference_calculator",
+        "//mediapipe/calculators/tflite:tflite_tensors_to_classification_calculator",
         "//mediapipe/calculators/tflite:tflite_tensors_to_floats_calculator",
         "//mediapipe/calculators/tflite:tflite_tensors_to_landmarks_calculator",
         "//mediapipe/calculators/util:detections_to_rects_calculator",
@@ -85,6 +88,7 @@ mediapipe_simple_subgraph(
         "//mediapipe/calculators/util:landmarks_to_render_data_calculator",
         "//mediapipe/calculators/util:rect_transformation_calculator",
         "//mediapipe/calculators/util:thresholding_calculator",
+        "//mediapipe/graphs/hand_tracking/calculators:hand_detections_to_rects_calculator",
     ],
 )
 
@@ -107,6 +111,7 @@ mediapipe_simple_subgraph(
     deps = [
         "//mediapipe/calculators/util:annotation_overlay_calculator",
         "//mediapipe/calculators/util:detections_to_render_data_calculator",
+        "//mediapipe/calculators/util:labels_to_render_data_calculator",
         "//mediapipe/calculators/util:landmarks_to_render_data_calculator",
         "//mediapipe/calculators/util:rect_to_render_data_calculator",
     ],
@@ -174,12 +179,15 @@ mediapipe_simple_subgraph(
     graph = "hand_landmark_gpu.pbtxt",
     register_as = "HandLandmarkSubgraph",
     deps = [
+        "//mediapipe/calculators/core:split_normalized_landmark_list_calculator",
         "//mediapipe/calculators/core:split_vector_calculator",
         "//mediapipe/calculators/image:image_cropping_calculator",
         "//mediapipe/calculators/image:image_properties_calculator",
         "//mediapipe/calculators/image:image_transformation_calculator",
         "//mediapipe/calculators/tflite:tflite_converter_calculator",
+        "//mediapipe/calculators/tflite:tflite_custom_op_resolver_calculator",
         "//mediapipe/calculators/tflite:tflite_inference_calculator",
+        "//mediapipe/calculators/tflite:tflite_tensors_to_classification_calculator",
         "//mediapipe/calculators/tflite:tflite_tensors_to_floats_calculator",
         "//mediapipe/calculators/tflite:tflite_tensors_to_landmarks_calculator",
         "//mediapipe/calculators/util:detections_to_rects_calculator",
@@ -188,6 +196,7 @@ mediapipe_simple_subgraph(
         "//mediapipe/calculators/util:landmarks_to_detection_calculator",
         "//mediapipe/calculators/util:rect_transformation_calculator",
         "//mediapipe/calculators/util:thresholding_calculator",
+        "//mediapipe/graphs/hand_tracking/calculators:hand_detections_to_rects_calculator",
     ],
 )
 
@@ -210,6 +219,7 @@ mediapipe_simple_subgraph(
     deps = [
         "//mediapipe/calculators/util:annotation_overlay_calculator",
         "//mediapipe/calculators/util:detections_to_render_data_calculator",
+        "//mediapipe/calculators/util:labels_to_render_data_calculator",
         "//mediapipe/calculators/util:landmarks_to_render_data_calculator",
         "//mediapipe/calculators/util:rect_to_render_data_calculator",
     ],

mediapipe/graphs/hand_tracking/subgraphs/hand_landmark_cpu.pbtxt

@@ -7,6 +7,8 @@ input_stream: "NORM_RECT:hand_rect"
 output_stream: "LANDMARKS:hand_landmarks"
 output_stream: "NORM_RECT:hand_rect_for_next_frame"
 output_stream: "PRESENCE:hand_presence"
+output_stream: "PRESENCE_SCORE:hand_presence_score"
+output_stream: "HANDEDNESS:handedness"
 
 # Crops the rectangle that contains a hand from the input image.
 node {
@@ -38,6 +40,13 @@ node: {
   }
 }
 
+# Generates a single side packet containing a TensorFlow Lite op resolver that
+# supports custom ops needed by the model used in this graph.
+node {
+  calculator: "TfLiteCustomOpResolverCalculator"
+  output_side_packet: "op_resolver"
+}
+
 # Converts the transformed input image on CPU into an image tensor stored in
 # TfliteTensor. The zero_center option is set to false to normalize the
 # pixel values to [0.f, 1.f].
@@ -59,6 +68,7 @@ node {
   calculator: "TfLiteInferenceCalculator"
   input_stream: "TENSORS:image_tensor"
   output_stream: "TENSORS:output_tensors"
+  input_side_packet: "CUSTOM_OP_RESOLVER:op_resolver"
   node_options: {
     [type.googleapis.com/mediapipe.TfLiteInferenceCalculatorOptions] {
       model_path: "mediapipe/models/hand_landmark.tflite"
@@ -73,10 +83,12 @@ node {
   input_stream: "output_tensors"
   output_stream: "landmark_tensors"
   output_stream: "hand_flag_tensor"
+  output_stream: "handedness_tensor"
   node_options: {
     [type.googleapis.com/mediapipe.SplitVectorCalculatorOptions] {
       ranges: { begin: 0 end: 1 }
       ranges: { begin: 1 end: 2 }
+      ranges: { begin: 2 end: 3 }
     }
   }
 }
@@ -89,6 +101,21 @@ node {
   output_stream: "FLOAT:hand_presence_score"
 }
 
+# Converts the handedness tensor into a float as the score of the handedness
+# binary classifciation.
+node {
+  calculator: "TfLiteTensorsToClassificationCalculator"
+  input_stream: "TENSORS:handedness_tensor"
+  output_stream: "CLASSIFICATIONS:handedness"
+  node_options: {
+    [type.googleapis.com/mediapipe.TfLiteTensorsToClassificationCalculatorOptions] {
+      top_k: 1
+      label_map_path: "mediapipe/models/handedness.txt"
+      binary_classification: true
+    }
+  }
+}
+
 # Applies a threshold to the confidence score to determine whether a hand is
 # present.
 node {
@@ -144,33 +171,49 @@ node {
   output_stream: "SIZE:image_size"
 }
 
-# Converts hand landmarks to a detection that tightly encloses all landmarks.
+# Extracts a subset of the hand landmarks that are relatively more stable across
+# frames (e.g. comparing to finger tips) for computing the bounding box. The box
+# will later be expanded to contain the entire hand. In this approach, it is
+# more robust to drastically changing hand size.
+# The landmarks extracted are: wrist, MCP/PIP of five fingers.
 node {
-  calculator: "LandmarksToDetectionCalculator"
-  input_stream: "NORM_LANDMARKS:hand_landmarks"
-  output_stream: "DETECTION:hand_detection"
-}
-
-# Converts the hand detection into a rectangle (normalized by image size)
-# that encloses the hand and is rotated such that the line connecting center of
-# the wrist and MCP of the middle finger is aligned with the Y-axis of the
-# rectangle.
-node {
-  calculator: "DetectionsToRectsCalculator"
-  input_stream: "DETECTION:hand_detection"
-  input_stream: "IMAGE_SIZE:image_size"
-  output_stream: "NORM_RECT:hand_rect_from_landmarks"
+  calculator: "SplitNormalizedLandmarkListCalculator"
+  input_stream: "hand_landmarks"
+  output_stream: "partial_landmarks"
   node_options: {
-    [type.googleapis.com/mediapipe.DetectionsToRectsCalculatorOptions] {
-      rotation_vector_start_keypoint_index: 0  # Center of wrist.
-      rotation_vector_end_keypoint_index: 9  # MCP of middle finger.
-      rotation_vector_target_angle_degrees: 90
+    [type.googleapis.com/mediapipe.SplitVectorCalculatorOptions] {
+      ranges: { begin: 0 end: 4 }
+      ranges: { begin: 5 end: 7 }
+      ranges: { begin: 9 end: 11 }
+      ranges: { begin: 13 end: 15 }
+      ranges: { begin: 17 end: 19 }
+      combine_outputs: true
     }
   }
 }
 
-# Expands the hand rectangle so that in the next video frame it's likely to
-# still contain the hand even with some motion.
+# Converts hand landmarks to a detection that tightly encloses all landmarks.
+node {
+  calculator: "LandmarksToDetectionCalculator"
+  input_stream: "NORM_LANDMARKS:partial_landmarks"
+  output_stream: "DETECTION:hand_detection"
+}
+
+# Converts the hand detection into a rectangle (normalized by image size)
+# that encloses the hand. The calculator uses a subset of all hand landmarks
+# extracted from SplitNormalizedLandmarkListCalculator above to
+# calculate the bounding box and the rotation of the output rectangle. Please
+# see the comments in the calculator for more detail.
+node {
+  calculator: "HandDetectionsToRectsCalculator"
+  input_stream: "DETECTION:hand_detection"
+  input_stream: "IMAGE_SIZE:image_size"
+  output_stream: "NORM_RECT:hand_rect_from_landmarks"
+}
+
+# Expands the hand rectangle so that the box contains the entire hand and it's
+# big enough so that it's likely to still contain the hand even with some motion
+# in the next video frame .
 node {
   calculator: "RectTransformationCalculator"
   input_stream: "NORM_RECT:hand_rect_from_landmarks"
@@ -178,8 +221,9 @@ node {
   output_stream: "hand_rect_for_next_frame"
   node_options: {
     [type.googleapis.com/mediapipe.RectTransformationCalculatorOptions] {
-      scale_x: 1.6
-      scale_y: 1.6
+      scale_x: 2.1
+      scale_y: 2.1
+      shift_y: -0.1
       square_long: true
     }
   }

mediapipe/graphs/hand_tracking/subgraphs/hand_landmark_gpu.pbtxt

@@ -7,6 +7,8 @@ input_stream: "NORM_RECT:hand_rect"
 output_stream: "LANDMARKS:hand_landmarks"
 output_stream: "NORM_RECT:hand_rect_for_next_frame"
 output_stream: "PRESENCE:hand_presence"
+output_stream: "PRESENCE_SCORE:hand_presence_score"
+output_stream: "HANDEDNESS:handedness"
 
 # Crops the rectangle that contains a hand from the input image.
 node {
@@ -38,6 +40,18 @@ node: {
   }
 }
 
+# Generates a single side packet containing a TensorFlow Lite op resolver that
+# supports custom ops needed by the model used in this graph.
+node {
+  calculator: "TfLiteCustomOpResolverCalculator"
+  output_side_packet: "op_resolver"
+  node_options: {
+    [type.googleapis.com/mediapipe.TfLiteCustomOpResolverCalculatorOptions] {
+      use_gpu: true
+    }
+  }
+}
+
 # Converts the transformed input image on GPU into an image tensor stored as a
 # TfLiteTensor.
 node {
@@ -58,6 +72,7 @@ node {
   calculator: "TfLiteInferenceCalculator"
   input_stream: "TENSORS_GPU:image_tensor"
   output_stream: "TENSORS:output_tensors"
+  input_side_packet: "CUSTOM_OP_RESOLVER:op_resolver"
   node_options: {
     [type.googleapis.com/mediapipe.TfLiteInferenceCalculatorOptions] {
       model_path: "mediapipe/models/hand_landmark.tflite"
@@ -72,10 +87,12 @@ node {
   input_stream: "output_tensors"
   output_stream: "landmark_tensors"
   output_stream: "hand_flag_tensor"
+  output_stream: "handedness_tensor"
   node_options: {
     [type.googleapis.com/mediapipe.SplitVectorCalculatorOptions] {
       ranges: { begin: 0 end: 1 }
       ranges: { begin: 1 end: 2 }
+      ranges: { begin: 2 end: 3 }
     }
   }
 }
@@ -88,6 +105,21 @@ node {
   output_stream: "FLOAT:hand_presence_score"
 }
 
+# Converts the handedness tensor into a float as the score of the handedness
+# binary classifciation.
+node {
+  calculator: "TfLiteTensorsToClassificationCalculator"
+  input_stream: "TENSORS:handedness_tensor"
+  output_stream: "CLASSIFICATIONS:handedness"
+  node_options: {
+    [type.googleapis.com/mediapipe.TfLiteTensorsToClassificationCalculatorOptions] {
+      top_k: 1
+      label_map_path: "mediapipe/models/handedness.txt"
+      binary_classification: true
+    }
+  }
+}
+
 # Applies a threshold to the confidence score to determine whether a hand is
 # present.
 node {
@@ -143,33 +175,46 @@ node {
   output_stream: "SIZE:image_size"
 }
 
-# Converts hand landmarks to a detection that tightly encloses all landmarks.
+# Extracts a subset of the hand landmarks that are relatively more stable across
+# frames (e.g. comparing to finger tips) for computing the bounding box. The box
+# will later be expanded to contain the entire hand. In this approach, it is
+# more robust to drastically changing hand size.
+# The landmarks extracted are: wrist, MCP/PIP of five fingers.
 node {
-  calculator: "LandmarksToDetectionCalculator"
-  input_stream: "NORM_LANDMARKS:hand_landmarks"
-  output_stream: "DETECTION:hand_detection"
-}
-
-# Converts the hand detection into a rectangle (normalized by image size)
-# that encloses the hand and is rotated such that the line connecting center of
-# the wrist and MCP of the middle finger is aligned with the Y-axis of the
-# rectangle.
-node {
-  calculator: "DetectionsToRectsCalculator"
-  input_stream: "DETECTION:hand_detection"
-  input_stream: "IMAGE_SIZE:image_size"
-  output_stream: "NORM_RECT:hand_rect_from_landmarks"
+  calculator: "SplitNormalizedLandmarkListCalculator"
+  input_stream: "hand_landmarks"
+  output_stream: "partial_landmarks"
   node_options: {
-    [type.googleapis.com/mediapipe.DetectionsToRectsCalculatorOptions] {
-      rotation_vector_start_keypoint_index: 0  # Center of wrist.
-      rotation_vector_end_keypoint_index: 9  # MCP of middle finger.
-      rotation_vector_target_angle_degrees: 90
+    [type.googleapis.com/mediapipe.SplitVectorCalculatorOptions] {
+      ranges: { begin: 0 end: 4 }
+      ranges: { begin: 5 end: 7 }
+      ranges: { begin: 9 end: 11 }
+      ranges: { begin: 13 end: 15 }
+      ranges: { begin: 17 end: 19 }
+      combine_outputs: true
     }
   }
 }
 
-# Expands the hand rectangle so that in the next video frame it's likely to
-# still contain the hand even with some motion.
+# Converts hand landmarks to a detection that tightly encloses all landmarks.
+node {
+  calculator: "LandmarksToDetectionCalculator"
+  input_stream: "NORM_LANDMARKS:partial_landmarks"
+  output_stream: "DETECTION:hand_detection"
+}
+
+# Converts the hand detection into a rectangle (normalized by image size)
+# that encloses the hand.
+node {
+  calculator: "HandDetectionsToRectsCalculator"
+  input_stream: "DETECTION:hand_detection"
+  input_stream: "IMAGE_SIZE:image_size"
+  output_stream: "NORM_RECT:hand_rect_from_landmarks"
+}
+
+# Expands the hand rectangle so that the box contains the entire hand and it's
+# big enough so that it's likely to still contain the hand even with some motion
+# in the next video frame .
 node {
   calculator: "RectTransformationCalculator"
   input_stream: "NORM_RECT:hand_rect_from_landmarks"
@@ -177,8 +222,9 @@ node {
   output_stream: "hand_rect_for_next_frame"
   node_options: {
     [type.googleapis.com/mediapipe.RectTransformationCalculatorOptions] {
-      scale_x: 1.6
-      scale_y: 1.6
+      scale_x: 2.1
+      scale_y: 2.1
+      shift_y: -0.1
       square_long: true
     }
   }

mediapipe/graphs/hand_tracking/subgraphs/renderer_cpu.pbtxt

@@ -6,8 +6,28 @@ input_stream: "IMAGE:input_image"
 input_stream: "DETECTIONS:detections"
 input_stream: "LANDMARKS:landmarks"
 input_stream: "NORM_RECT:rect"
+input_stream: "HANDEDNESS:handedness"
 output_stream: "IMAGE:output_image"
 
+# Converts classification to drawing primitives for annotation overlay.
+node {
+  calculator: "LabelsToRenderDataCalculator"
+  input_stream: "CLASSIFICATIONS:handedness"
+  output_stream: "RENDER_DATA:handedness_render_data"
+  node_options: {
+    [type.googleapis.com/mediapipe.LabelsToRenderDataCalculatorOptions]: {
+      color { r: 255 g: 0 b: 0 }
+      thickness: 10.0
+      font_height_px: 50
+      horizontal_offset_px: 100
+      vertical_offset_px: 100
+
+      max_num_labels: 1
+      location: TOP_LEFT
+    }
+  }
+}
+
 # Converts detections to drawing primitives for annotation overlay.
 node {
   calculator: "DetectionsToRenderDataCalculator"
@@ -97,6 +117,7 @@ node {
   input_stream: "IMAGE:input_image"
   input_stream: "detection_render_data"
   input_stream: "landmark_render_data"
+  input_stream: "handedness_render_data"
   input_stream: "rect_render_data"
   output_stream: "IMAGE:output_image"
 }

mediapipe/graphs/hand_tracking/subgraphs/renderer_gpu.pbtxt

@@ -6,8 +6,28 @@ input_stream: "IMAGE:input_image"
 input_stream: "DETECTIONS:detections"
 input_stream: "LANDMARKS:landmarks"
 input_stream: "NORM_RECT:rect"
+input_stream: "HANDEDNESS:handedness"
 output_stream: "IMAGE:output_image"
 
+# Converts classification to drawing primitives for annotation overlay.
+node {
+  calculator: "LabelsToRenderDataCalculator"
+  input_stream: "CLASSIFICATIONS:handedness"
+  output_stream: "RENDER_DATA:handedness_render_data"
+  node_options: {
+    [type.googleapis.com/mediapipe.LabelsToRenderDataCalculatorOptions]: {
+      color { r: 255 g: 0 b: 0 }
+      thickness: 10.0
+      font_height_px: 50
+      horizontal_offset_px: 200
+      vertical_offset_px: 100
+
+      max_num_labels: 1
+      location: TOP_LEFT
+    }
+  }
+}
+
 # Converts detections to drawing primitives for annotation overlay.
 node {
   calculator: "DetectionsToRenderDataCalculator"
@@ -97,6 +117,7 @@ node {
   input_stream: "IMAGE_GPU:input_image"
   input_stream: "detection_render_data"
   input_stream: "landmark_render_data"
+  input_stream: "handedness_render_data"
   input_stream: "rect_render_data"
   output_stream: "IMAGE_GPU:output_image"
 }

mediapipe/graphs/template_matching/index_building.pbtxt

@@ -19,15 +19,28 @@ node {
   output_stream: "image_frame"
 }
 
+node: {
+  calculator: "ImageTransformationCalculator"
+  input_stream: "IMAGE:image_frame"
+  output_stream: "IMAGE:scaled_image_frame"
+  node_options: {
+    [type.googleapis.com/mediapipe.ImageTransformationCalculatorOptions] {
+      output_width: 320
+      output_height: 320
+      scale_mode: FILL_AND_CROP
+    }
+  }
+}
+
 node {
   calculator: "ImagePropertiesCalculator"
-  input_stream: "IMAGE:image_frame"
+  input_stream: "IMAGE:scaled_image_frame"
   output_stream: "SIZE:input_video_size"
 }
 
 node {
   calculator: "FeatureDetectorCalculator"
-  input_stream: "IMAGE:image_frame"
+  input_stream: "IMAGE:scaled_image_frame"
   output_stream: "FEATURES:features"
   output_stream: "LANDMARKS:landmarks"
   output_stream: "PATCHES:patches"

mediapipe/graphs/template_matching/template_matching_desktop.pbtxt

@@ -16,15 +16,28 @@ node {
   output_stream: "VIDEO_PRESTREAM:input_video_header"
 }
 
+node: {
+  calculator: "ImageTransformationCalculator"
+  input_stream: "IMAGE:input_video"
+  output_stream: "IMAGE:scaled_input_video"
+  node_options: {
+    [type.googleapis.com/mediapipe.ImageTransformationCalculatorOptions] {
+      output_width: 640
+      output_height: 640
+      scale_mode: FILL_AND_CROP
+    }
+  }
+}
+
 node {
   calculator: "ImagePropertiesCalculator"
-  input_stream: "IMAGE:input_video"
+  input_stream: "IMAGE:scaled_input_video"
   output_stream: "SIZE:input_video_size"
 }
 
 node {
   calculator: "FeatureDetectorCalculator"
-  input_stream: "IMAGE:input_video"
+  input_stream: "IMAGE:scaled_input_video"
   output_stream: "FEATURES:features"
   output_stream: "LANDMARKS:landmarks"
   output_stream: "PATCHES:patches"

mediapipe/graphs/template_matching/template_matching_mobile_cpu.pbtxt

@@ -32,8 +32,9 @@ node: {
   output_stream: "IMAGE:transformed_input_video_cpu"
   node_options: {
     [type.googleapis.com/mediapipe.ImageTransformationCalculatorOptions] {
-      output_width: 480
+      output_width: 640
       output_height: 640
+      scale_mode: FILL_AND_CROP
     }
   }
 }