Project import generated by Copybara.

GitOrigin-RevId: d0039a576e2db9c0fcefffd26a527df74cbe145b
2020-04-21 22:43:01 -04:00
parent 024f7bf0f1
commit 7bad8fce62
45 changed files with 1566 additions and 227 deletions
@@ -0,0 +1,67 @@
+# Copyright 2019 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load(
+    "//mediapipe/framework/tool:mediapipe_graph.bzl",
+    "mediapipe_binary_graph",
+)
+
+licenses(["notice"])  # Apache 2.0
+
+package(default_visibility = ["//visibility:public"])
+
+cc_library(
+    name = "template_matching_deps",
+    deps = [
+        "//mediapipe/calculators/image:feature_detector_calculator",
+        "//mediapipe/calculators/image:image_properties_calculator",
+        "//mediapipe/calculators/image:image_transformation_calculator",
+        "//mediapipe/calculators/tflite:tflite_converter_calculator",
+        "//mediapipe/calculators/tflite:tflite_inference_calculator",
+        "//mediapipe/calculators/tflite:tflite_tensors_to_floats_calculator",
+        "//mediapipe/calculators/util:annotation_overlay_calculator",
+        "//mediapipe/calculators/util:landmarks_to_render_data_calculator",
+        "//mediapipe/calculators/util:timed_box_list_id_to_label_calculator",
+        "//mediapipe/calculators/util:timed_box_list_to_render_data_calculator",
+        "//mediapipe/calculators/video:box_detector_calculator",
+    ],
+)
+
+cc_library(
+    name = "desktop_calculators",
+    deps = [
+        ":template_matching_deps",
+        "//mediapipe/calculators/image:opencv_encoded_image_to_image_frame_calculator",
+        "//mediapipe/calculators/util:local_file_pattern_contents_calculator",
+        "//mediapipe/calculators/video:opencv_video_decoder_calculator",
+        "//mediapipe/calculators/video:opencv_video_encoder_calculator",
+    ],
+)
+
+cc_library(
+    name = "mobile_calculators",
+    deps = [
+        ":template_matching_deps",
+        "//mediapipe/calculators/core:flow_limiter_calculator",
+        "//mediapipe/calculators/image:image_transformation_calculator",
+        "//mediapipe/gpu:gpu_buffer_to_image_frame_calculator",
+    ],
+)
+
+mediapipe_binary_graph(
+    name = "mobile_cpu_binary_graph",
+    graph = "template_matching_mobile_cpu.pbtxt",
+    output_name = "mobile_cpu.binarypb",
+    deps = [":mobile_calculators"],
+)
@@ -0,0 +1,79 @@
+# MediaPipe graph that build feature descriptors index for specific target.
+
+# max_queue_size limits the number of packets enqueued on any input stream
+# by throttling inputs to the graph. This makes the graph only process one
+# frame per time.
+max_queue_size: 1
+
+# Decodes an input video file into images and a video header.
+node {
+  calculator: "LocalFilePatternContentsCalculator"
+  input_side_packet: "FILE_DIRECTORY:file_directory"
+  input_side_packet: "FILE_SUFFIX:file_suffix"
+  output_stream: "CONTENTS:encoded_image"
+}
+
+node {
+  calculator: "OpenCvEncodedImageToImageFrameCalculator"
+  input_stream: "encoded_image"
+  output_stream: "image_frame"
+}
+
+node {
+  calculator: "ImagePropertiesCalculator"
+  input_stream: "IMAGE:image_frame"
+  output_stream: "SIZE:input_video_size"
+}
+
+node {
+  calculator: "FeatureDetectorCalculator"
+  input_stream: "IMAGE:image_frame"
+  output_stream: "FEATURES:features"
+  output_stream: "LANDMARKS:landmarks"
+  output_stream: "PATCHES:patches"
+  node_options: {
+    [type.googleapis.com/mediapipe.FeatureDetectorCalculatorOptions] {
+      max_features: 400
+    }
+  }
+}
+
+# input tensors: 200*32*32*1 float
+# output tensors: 200*40 float, only first keypoint.size()*40 is knift features,
+# rest is padded by zero.
+node {
+  calculator: "TfLiteInferenceCalculator"
+  input_stream: "TENSORS:patches"
+  output_stream: "TENSORS:knift_feature_tensors"
+  input_stream_handler {
+    input_stream_handler: "DefaultInputStreamHandler"
+  }
+  node_options: {
+    [type.googleapis.com/mediapipe.TfLiteInferenceCalculatorOptions] {
+      model_path: "mediapipe/models/knift_float_400.tflite"
+    }
+  }
+}
+
+node {
+  calculator: "TfLiteTensorsToFloatsCalculator"
+  input_stream: "TENSORS:knift_feature_tensors"
+  output_stream: "FLOATS:knift_feature_floats"
+}
+
+node {
+  calculator: "BoxDetectorCalculator"
+  input_side_packet: "OUTPUT_INDEX_FILENAME:output_index_filename"
+  input_stream: "FEATURES:features"
+  input_stream: "IMAGE_SIZE:input_video_size"
+  input_stream: "DESCRIPTORS:knift_feature_floats"
+
+  node_options: {
+    [type.googleapis.com/mediapipe.BoxDetectorCalculatorOptions] {
+      detector_options {
+        index_type: OPENCV_BF
+        detect_every_n_frame: 1
+      }
+    }
+  }
+}
@@ -0,0 +1,128 @@
+# MediaPipe graph that performs object detection on desktop with TensorFlow Lite
+# on CPU.
+# Used in the example in
+# mediapipe/examples/desktop/template_matching:template_matching_tflite
+
+# max_queue_size limits the number of packets enqueued on any input stream
+# by throttling inputs to the graph. This makes the graph only process one
+# frame per time.
+max_queue_size: 1
+
+# Decodes an input video file into images and a video header.
+node {
+  calculator: "OpenCvVideoDecoderCalculator"
+  input_side_packet: "INPUT_FILE_PATH:input_video_path"
+  output_stream: "VIDEO:input_video"
+  output_stream: "VIDEO_PRESTREAM:input_video_header"
+}
+
+node {
+  calculator: "ImagePropertiesCalculator"
+  input_stream: "IMAGE:input_video"
+  output_stream: "SIZE:input_video_size"
+}
+
+node {
+  calculator: "FeatureDetectorCalculator"
+  input_stream: "IMAGE:input_video"
+  output_stream: "FEATURES:features"
+  output_stream: "LANDMARKS:landmarks"
+  output_stream: "PATCHES:patches"
+}
+
+# input tensors: 200*32*32*1 float
+# output tensors: 200*40 float, only first keypoint.size()*40 is knift features,
+# rest is padded by zero.
+node {
+  calculator: "TfLiteInferenceCalculator"
+  input_stream: "TENSORS:patches"
+  output_stream: "TENSORS:knift_feature_tensors"
+  node_options: {
+    [type.googleapis.com/mediapipe.TfLiteInferenceCalculatorOptions] {
+      model_path: "mediapipe/models/knift_float.tflite"
+    }
+  }
+}
+
+node {
+  calculator: "TfLiteTensorsToFloatsCalculator"
+  input_stream: "TENSORS:knift_feature_tensors"
+  output_stream: "FLOATS:knift_feature_floats"
+}
+
+node {
+  calculator: "BoxDetectorCalculator"
+  input_stream: "FEATURES:features"
+  input_stream: "IMAGE_SIZE:input_video_size"
+  input_stream: "DESCRIPTORS:knift_feature_floats"
+  output_stream: "BOXES:detections"
+
+  node_options: {
+    [type.googleapis.com/mediapipe.BoxDetectorCalculatorOptions] {
+      detector_options {
+        index_type: OPENCV_BF
+        detect_every_n_frame: 1
+      }
+      index_proto_filename: "mediapipe/models/knift_index.pb"
+    }
+  }
+}
+
+node {
+  calculator: "TimedBoxListIdToLabelCalculator"
+  input_stream: "detections"
+  output_stream: "labeled_detections"
+  node_options: {
+    [type.googleapis.com/mediapipe.TimedBoxListIdToLabelCalculatorOptions] {
+      label_map_path: "mediapipe/models/knift_labelmap.txt"
+    }
+  }
+}
+
+node {
+  calculator: "TimedBoxListToRenderDataCalculator"
+  input_stream: "BOX_LIST:labeled_detections"
+  output_stream: "RENDER_DATA:box_render_data"
+  node_options: {
+    [type.googleapis.com/mediapipe.TimedBoxListToRenderDataCalculatorOptions] {
+      box_color { r: 255 g: 0 b: 0 }
+      thickness: 5.0
+    }
+  }
+}
+
+node {
+  calculator: "LandmarksToRenderDataCalculator"
+  input_stream: "NORM_LANDMARKS:landmarks"
+  output_stream: "RENDER_DATA:landmarks_render_data"
+  node_options: {
+    [type.googleapis.com/mediapipe.LandmarksToRenderDataCalculatorOptions] {
+      landmark_color { r: 0 g: 255 b: 0 }
+      thickness: 2.0
+    }
+  }
+}
+
+# Draws annotations and overlays them on top of the input images.
+node {
+  calculator: "AnnotationOverlayCalculator"
+  input_stream: "IMAGE:input_video"
+  input_stream: "box_render_data"
+  input_stream: "landmarks_render_data"
+  output_stream: "IMAGE:output_video"
+}
+
+# Encodes the annotated images into a video file, adopting properties specified
+# in the input video header, e.g., video framerate.
+node {
+  calculator: "OpenCvVideoEncoderCalculator"
+  input_stream: "VIDEO:output_video"
+  input_stream: "VIDEO_PRESTREAM:input_video_header"
+  input_side_packet: "OUTPUT_FILE_PATH:output_video_path"
+  node_options: {
+    [type.googleapis.com/mediapipe.OpenCvVideoEncoderCalculatorOptions]: {
+      codec: "avc1"
+      video_format: "mp4"
+    }
+  }
+}
@@ -0,0 +1,136 @@
+# MediaPipe graph that performs template matching with TensorFlow Lite on CPU.
+# Used in the examples in
+# mediapipe/examples/android/src/java/com/mediapipe/apps/templatematchingcpu
+
+# Images on GPU coming into and out of the graph.
+input_stream: "input_video"
+output_stream: "output_video"
+
+# Throttles the images flowing downstream for flow control.
+node {
+  calculator: "FlowLimiterCalculator"
+  input_stream: "input_video"
+  input_stream: "FINISHED:detections"
+  input_stream_info: {
+    tag_index: "FINISHED"
+    back_edge: true
+  }
+  output_stream: "throttled_input_video"
+}
+
+# Transfers the input image from GPU to CPU memory.
+node: {
+  calculator: "GpuBufferToImageFrameCalculator"
+  input_stream: "throttled_input_video"
+  output_stream: "input_video_cpu"
+}
+
+# Transforms the input image on CPU to a 480x640 image.
+node: {
+  calculator: "ImageTransformationCalculator"
+  input_stream: "IMAGE:input_video_cpu"
+  output_stream: "IMAGE:transformed_input_video_cpu"
+  node_options: {
+    [type.googleapis.com/mediapipe.ImageTransformationCalculatorOptions] {
+      output_width: 480
+      output_height: 640
+    }
+  }
+}
+
+node {
+  calculator: "ImagePropertiesCalculator"
+  input_stream: "IMAGE:transformed_input_video_cpu"
+  output_stream: "SIZE:input_video_size"
+}
+
+node {
+  calculator: "FeatureDetectorCalculator"
+  input_stream: "IMAGE:transformed_input_video_cpu"
+  output_stream: "FEATURES:features"
+  output_stream: "LANDMARKS:landmarks"
+  output_stream: "PATCHES:patches"
+}
+
+# input tensors: 200*32*32*1 float
+# output tensors: 200*40 float, only first keypoint.size()*40 is knift features,
+# rest is padded by zero.
+node {
+  calculator: "TfLiteInferenceCalculator"
+  input_stream: "TENSORS:patches"
+  output_stream: "TENSORS:knift_feature_tensors"
+  node_options: {
+    [type.googleapis.com/mediapipe.TfLiteInferenceCalculatorOptions] {
+      model_path: "mediapipe/models/knift_float.tflite"
+      delegate { xnnpack {} }
+    }
+  }
+}
+
+node {
+  calculator: "TfLiteTensorsToFloatsCalculator"
+  input_stream: "TENSORS:knift_feature_tensors"
+  output_stream: "FLOATS:knift_feature_floats"
+}
+
+node {
+  calculator: "BoxDetectorCalculator"
+  input_stream: "FEATURES:features"
+  input_stream: "IMAGE_SIZE:input_video_size"
+  input_stream: "DESCRIPTORS:knift_feature_floats"
+  output_stream: "BOXES:detections"
+
+  node_options: {
+    [type.googleapis.com/mediapipe.BoxDetectorCalculatorOptions] {
+      detector_options {
+        index_type: OPENCV_BF
+        detect_every_n_frame: 1
+      }
+      index_proto_filename: "mediapipe/models/knift_index.pb"
+    }
+  }
+}
+
+node {
+  calculator: "TimedBoxListIdToLabelCalculator"
+  input_stream: "detections"
+  output_stream: "labeled_detections"
+  node_options: {
+    [type.googleapis.com/mediapipe.TimedBoxListIdToLabelCalculatorOptions] {
+      label_map_path: "mediapipe/models/knift_labelmap.txt"
+    }
+  }
+}
+
+node {
+  calculator: "TimedBoxListToRenderDataCalculator"
+  input_stream: "BOX_LIST:labeled_detections"
+  output_stream: "RENDER_DATA:box_render_data"
+  node_options: {
+    [type.googleapis.com/mediapipe.TimedBoxListToRenderDataCalculatorOptions] {
+      box_color { r: 255 g: 0 b: 0 }
+      thickness: 5.0
+    }
+  }
+}
+
+node {
+  calculator: "LandmarksToRenderDataCalculator"
+  input_stream: "NORM_LANDMARKS:landmarks"
+  output_stream: "RENDER_DATA:landmarks_render_data"
+  node_options: {
+    [type.googleapis.com/mediapipe.LandmarksToRenderDataCalculatorOptions] {
+      landmark_color { r: 0 g: 255 b: 0 }
+      thickness: 2.0
+    }
+  }
+}
+
+# Draws annotations and overlays them on top of the input images.
+node {
+  calculator: "AnnotationOverlayCalculator"
+  input_stream: "IMAGE_GPU:throttled_input_video"
+  input_stream: "box_render_data"
+  input_stream: "landmarks_render_data"
+  output_stream: "IMAGE_GPU:output_video"
+}