Project import generated by Copybara.

GitOrigin-RevId: 53a42bf7ad836321123cb7b6c80b0f2e13fbf83e

mediapipe/calculators/core/BUILD

@@ -230,6 +230,7 @@ cc_library(
         "//mediapipe/framework:packet",
         "//mediapipe/framework/formats:detection_cc_proto",
         "//mediapipe/framework/formats:landmark_cc_proto",
+        "//mediapipe/framework/formats:matrix",
         "//mediapipe/framework/formats:rect_cc_proto",
         "//mediapipe/framework/port:integral_types",
         "//mediapipe/framework/port:ret_check",
@@ -257,6 +258,7 @@ cc_library(
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework/port:status",
         "//mediapipe/util:render_data_cc_proto",
+        "@org_tensorflow//tensorflow/lite:framework",
     ],
     alwayslink = 1,
 )
@@ -779,6 +781,7 @@ cc_library(
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework/formats:landmark_cc_proto",
         "//mediapipe/framework/formats:rect_cc_proto",
+        "//mediapipe/framework/formats:matrix",
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework/port:status",
         "//mediapipe/util:resource_util",

mediapipe/calculators/core/begin_loop_calculator.cc

@@ -18,6 +18,7 @@
 
 #include "mediapipe/framework/formats/detection.pb.h"
 #include "mediapipe/framework/formats/landmark.pb.h"
+#include "mediapipe/framework/formats/matrix.h"
 #include "mediapipe/framework/formats/rect.pb.h"
 
 namespace mediapipe {
@@ -37,4 +38,8 @@ typedef BeginLoopCalculator<std::vector<::mediapipe::Detection>>
     BeginLoopDetectionCalculator;
 REGISTER_CALCULATOR(BeginLoopDetectionCalculator);
 
+// A calculator to process std::vector<Matrix>.
+typedef BeginLoopCalculator<std::vector<Matrix>> BeginLoopMatrixCalculator;
+REGISTER_CALCULATOR(BeginLoopMatrixCalculator);
+
 }  // namespace mediapipe

mediapipe/calculators/core/end_loop_calculator.cc

@@ -20,6 +20,7 @@
 #include "mediapipe/framework/formats/landmark.pb.h"
 #include "mediapipe/framework/formats/rect.pb.h"
 #include "mediapipe/util/render_data.pb.h"
+#include "tensorflow/lite/interpreter.h"
 
 namespace mediapipe {
 
@@ -42,4 +43,7 @@ typedef EndLoopCalculator<std::vector<::mediapipe::ClassificationList>>
     EndLoopClassificationListCalculator;
 REGISTER_CALCULATOR(EndLoopClassificationListCalculator);
 
+typedef EndLoopCalculator<std::vector<TfLiteTensor>> EndLoopTensorCalculator;
+REGISTER_CALCULATOR(EndLoopTensorCalculator);
+
 }  // namespace mediapipe

mediapipe/calculators/core/split_vector_calculator.cc

@@ -18,6 +18,7 @@
 
 #include "mediapipe/framework/formats/detection.pb.h"
 #include "mediapipe/framework/formats/landmark.pb.h"
+#include "mediapipe/framework/formats/matrix.h"
 #include "mediapipe/framework/formats/rect.pb.h"
 #include "tensorflow/lite/interpreter.h"
 
@@ -57,6 +58,9 @@ typedef SplitVectorCalculator<::mediapipe::NormalizedRect, false>
     SplitNormalizedRectVectorCalculator;
 REGISTER_CALCULATOR(SplitNormalizedRectVectorCalculator);
 
+typedef SplitVectorCalculator<Matrix, false> SplitMatrixVectorCalculator;
+REGISTER_CALCULATOR(SplitMatrixVectorCalculator);
+
 #if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
 typedef SplitVectorCalculator<::tflite::gpu::gl::GlBuffer, true>
     MovableSplitGlBufferVectorCalculator;

mediapipe/calculators/image/BUILD

@@ -86,6 +86,14 @@ mediapipe_cc_proto_library(
     deps = [":opencv_image_encoder_calculator_proto"],
 )
 
+mediapipe_cc_proto_library(
+    name = "opencv_encoded_image_to_image_frame_calculator_cc_proto",
+    srcs = ["opencv_encoded_image_to_image_frame_calculator.proto"],
+    cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
+    visibility = ["//visibility:public"],
+    deps = [":opencv_encoded_image_to_image_frame_calculator_proto"],
+)
+
 mediapipe_cc_proto_library(
     name = "mask_overlay_calculator_cc_proto",
     srcs = ["mask_overlay_calculator.proto"],
@@ -172,6 +180,7 @@ cc_library(
     srcs = ["opencv_encoded_image_to_image_frame_calculator.cc"],
     visibility = ["//visibility:public"],
     deps = [
+        ":opencv_encoded_image_to_image_frame_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework/formats:image_frame_opencv",
         "//mediapipe/framework/port:opencv_imgcodecs",
@@ -557,6 +566,27 @@ proto_library(
     deps = ["//mediapipe/framework:calculator_proto"],
 )
 
+proto_library(
+    name = "opencv_encoded_image_to_image_frame_calculator_proto",
+    srcs = ["opencv_encoded_image_to_image_frame_calculator.proto"],
+    visibility = ["//visibility:public"],
+    deps = ["//mediapipe/framework:calculator_proto"],
+)
+
+proto_library(
+    name = "feature_detector_calculator_proto",
+    srcs = ["feature_detector_calculator.proto"],
+    deps = ["//mediapipe/framework:calculator_proto"],
+)
+
+mediapipe_cc_proto_library(
+    name = "feature_detector_calculator_cc_proto",
+    srcs = ["feature_detector_calculator.proto"],
+    cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
+    visibility = ["//visibility:public"],
+    deps = [":feature_detector_calculator_proto"],
+)
+
 cc_library(
     name = "mask_overlay_calculator",
     srcs = ["mask_overlay_calculator.cc"],
@@ -572,3 +602,30 @@ cc_library(
     ],
     alwayslink = 1,
 )
+
+cc_library(
+    name = "feature_detector_calculator",
+    srcs = ["feature_detector_calculator.cc"],
+    visibility = ["//mediapipe:__subpackages__"],
+    deps = [
+        ":feature_detector_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:image_frame",
+        "//mediapipe/framework/formats:image_frame_opencv",
+        "//mediapipe/framework/formats:landmark_cc_proto",
+        "//mediapipe/framework/formats:video_stream_header",
+        "//mediapipe/framework/port:integral_types",
+        "//mediapipe/framework/port:logging",
+        "//mediapipe/framework/port:opencv_core",
+        "//mediapipe/framework/port:opencv_features2d",
+        "//mediapipe/framework/port:opencv_imgproc",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/framework/port:threadpool",
+        "//mediapipe/framework/tool:options_util",
+        "@com_google_absl//absl/memory",
+        "@com_google_absl//absl/synchronization",
+        "@org_tensorflow//tensorflow/lite:framework",
+    ],
+    alwayslink = 1,
+)

mediapipe/calculators/image/feature_detector_calculator.cc

@@ -0,0 +1,210 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <memory>
+#include <vector>
+
+#include "absl/memory/memory.h"
+#include "absl/synchronization/blocking_counter.h"
+#include "mediapipe/calculators/image/feature_detector_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/image_frame.h"
+#include "mediapipe/framework/formats/image_frame_opencv.h"
+#include "mediapipe/framework/formats/landmark.pb.h"
+#include "mediapipe/framework/formats/video_stream_header.h"
+#include "mediapipe/framework/port/integral_types.h"
+#include "mediapipe/framework/port/logging.h"
+#include "mediapipe/framework/port/opencv_core_inc.h"
+#include "mediapipe/framework/port/opencv_features2d_inc.h"
+#include "mediapipe/framework/port/opencv_imgproc_inc.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/port/threadpool.h"
+#include "mediapipe/framework/tool/options_util.h"
+#include "tensorflow/lite/interpreter.h"
+
+namespace mediapipe {
+
+const char kOptionsTag[] = "OPTIONS";
+const int kPatchSize = 32;
+const int kNumThreads = 16;
+
+// A calculator to apply local feature detection.
+// Input stream:
+//   IMAGE: Input image frame of type ImageFrame from video stream.
+// Output streams:
+//   FEATURES: The detected keypoints from input image as vector<cv::KeyPoint>.
+//   PATCHES:  Optional output the extracted patches as vector<cv::Mat>
+class FeatureDetectorCalculator : public CalculatorBase {
+ public:
+  ~FeatureDetectorCalculator() override = default;
+
+  static ::mediapipe::Status GetContract(CalculatorContract* cc);
+
+  ::mediapipe::Status Open(CalculatorContext* cc) override;
+  ::mediapipe::Status Process(CalculatorContext* cc) override;
+
+ private:
+  FeatureDetectorCalculatorOptions options_;
+  cv::Ptr<cv::Feature2D> feature_detector_;
+  std::unique_ptr<::mediapipe::ThreadPool> pool_;
+
+  // Create image pyramid based on input image.
+  void ComputeImagePyramid(const cv::Mat& input_image,
+                           std::vector<cv::Mat>* image_pyramid);
+
+  // Extract the patch for single feature with image pyramid.
+  cv::Mat ExtractPatch(const cv::KeyPoint& feature,
+                       const std::vector<cv::Mat>& image_pyramid);
+};
+
+REGISTER_CALCULATOR(FeatureDetectorCalculator);
+
+::mediapipe::Status FeatureDetectorCalculator::GetContract(
+    CalculatorContract* cc) {
+  if (cc->Inputs().HasTag("IMAGE")) {
+    cc->Inputs().Tag("IMAGE").Set<ImageFrame>();
+  }
+  if (cc->Outputs().HasTag("FEATURES")) {
+    cc->Outputs().Tag("FEATURES").Set<std::vector<cv::KeyPoint>>();
+  }
+  if (cc->Outputs().HasTag("LANDMARKS")) {
+    cc->Outputs().Tag("LANDMARKS").Set<NormalizedLandmarkList>();
+  }
+  if (cc->Outputs().HasTag("PATCHES")) {
+    cc->Outputs().Tag("PATCHES").Set<std::vector<TfLiteTensor>>();
+  }
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status FeatureDetectorCalculator::Open(CalculatorContext* cc) {
+  options_ =
+      tool::RetrieveOptions(cc->Options(), cc->InputSidePackets(), kOptionsTag)
+          .GetExtension(FeatureDetectorCalculatorOptions::ext);
+  feature_detector_ = cv::ORB::create(
+      options_.max_features(), options_.scale_factor(),
+      options_.pyramid_level(), kPatchSize - 1, 0, 2, cv::ORB::FAST_SCORE);
+  pool_ = absl::make_unique<::mediapipe::ThreadPool>("ThreadPool", kNumThreads);
+  pool_->StartWorkers();
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status FeatureDetectorCalculator::Process(CalculatorContext* cc) {
+  const Timestamp& timestamp = cc->InputTimestamp();
+  if (timestamp == Timestamp::PreStream()) {
+    // Indicator packet.
+    return ::mediapipe::OkStatus();
+  }
+  InputStream* input_frame = &(cc->Inputs().Tag("IMAGE"));
+  cv::Mat input_view = formats::MatView(&input_frame->Get<ImageFrame>());
+  cv::Mat grayscale_view;
+  cv::cvtColor(input_view, grayscale_view, cv::COLOR_RGB2GRAY);
+
+  std::vector<cv::KeyPoint> keypoints;
+  feature_detector_->detect(grayscale_view, keypoints);
+  if (keypoints.size() > options_.max_features()) {
+    keypoints.resize(options_.max_features());
+  }
+
+  if (cc->Outputs().HasTag("FEATURES")) {
+    auto features_ptr = absl::make_unique<std::vector<cv::KeyPoint>>(keypoints);
+    cc->Outputs().Tag("FEATURES").Add(features_ptr.release(), timestamp);
+  }
+
+  if (cc->Outputs().HasTag("LANDMARKS")) {
+    auto landmarks_ptr = absl::make_unique<NormalizedLandmarkList>();
+    for (int j = 0; j < keypoints.size(); ++j) {
+      auto feature_landmark = landmarks_ptr->add_landmark();
+      feature_landmark->set_x(keypoints[j].pt.x / grayscale_view.cols);
+      feature_landmark->set_y(keypoints[j].pt.y / grayscale_view.rows);
+    }
+    cc->Outputs().Tag("LANDMARKS").Add(landmarks_ptr.release(), timestamp);
+  }
+
+  if (cc->Outputs().HasTag("PATCHES")) {
+    std::vector<cv::Mat> image_pyramid;
+    ComputeImagePyramid(grayscale_view, &image_pyramid);
+    std::vector<cv::Mat> patch_mat;
+    patch_mat.resize(keypoints.size());
+    absl::BlockingCounter counter(keypoints.size());
+    for (int i = 0; i < keypoints.size(); i++) {
+      pool_->Schedule(
+          [this, &image_pyramid, &keypoints, &patch_mat, i, &counter] {
+            patch_mat[i] = ExtractPatch(keypoints[i], image_pyramid);
+            counter.DecrementCount();
+          });
+    }
+    counter.Wait();
+    const int batch_size = options_.max_features();
+    auto patches = absl::make_unique<std::vector<TfLiteTensor>>();
+    TfLiteTensor tensor;
+    tensor.type = kTfLiteFloat32;
+    tensor.dims = TfLiteIntArrayCreate(4);
+    tensor.dims->data[0] = batch_size;
+    tensor.dims->data[1] = kPatchSize;
+    tensor.dims->data[2] = kPatchSize;
+    tensor.dims->data[3] = 1;
+    int num_bytes = batch_size * kPatchSize * kPatchSize * sizeof(float);
+    tensor.data.data = malloc(num_bytes);
+    tensor.bytes = num_bytes;
+    tensor.allocation_type = kTfLiteArenaRw;
+    float* tensor_buffer = tensor.data.f;
+    for (int i = 0; i < keypoints.size(); i++) {
+      for (int j = 0; j < patch_mat[i].rows; ++j) {
+        for (int k = 0; k < patch_mat[i].cols; ++k) {
+          *tensor_buffer++ = patch_mat[i].at<uchar>(j, k) / 128.0f - 1.0f;
+        }
+      }
+    }
+    for (int i = keypoints.size() * kPatchSize * kPatchSize; i < num_bytes / 4;
+         i++) {
+      *tensor_buffer++ = 0;
+    }
+
+    patches->emplace_back(tensor);
+    cc->Outputs().Tag("PATCHES").Add(patches.release(), timestamp);
+  }
+
+  return ::mediapipe::OkStatus();
+}
+
+void FeatureDetectorCalculator::ComputeImagePyramid(
+    const cv::Mat& input_image, std::vector<cv::Mat>* image_pyramid) {
+  cv::Mat tmp_image = input_image;
+  cv::Mat src_image = input_image;
+  for (int i = 0; i < options_.pyramid_level(); ++i) {
+    image_pyramid->push_back(src_image);
+    cv::resize(src_image, tmp_image, cv::Size(), 1.0f / options_.scale_factor(),
+               1.0f / options_.scale_factor());
+    src_image = tmp_image;
+  }
+}
+
+cv::Mat FeatureDetectorCalculator::ExtractPatch(
+    const cv::KeyPoint& feature, const std::vector<cv::Mat>& image_pyramid) {
+  cv::Mat img = image_pyramid[feature.octave];
+  float scale_factor = 1 / pow(options_.scale_factor(), feature.octave);
+  cv::Point2f center =
+      cv::Point2f(feature.pt.x * scale_factor, feature.pt.y * scale_factor);
+  cv::Mat rot = cv::getRotationMatrix2D(center, feature.angle, 1.0);
+  rot.at<double>(0, 2) += kPatchSize / 2 - center.x;
+  rot.at<double>(1, 2) += kPatchSize / 2 - center.y;
+  cv::Mat cropped_img;
+  // perform the affine transformation
+  cv::warpAffine(img, cropped_img, rot, cv::Size(kPatchSize, kPatchSize),
+                 cv::INTER_LINEAR);
+  return cropped_img;
+}
+
+}  // namespace mediapipe

mediapipe/calculators/image/feature_detector_calculator.proto

@@ -0,0 +1,24 @@
+// Options for FeatureDetectorCalculator
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+
+message FeatureDetectorCalculatorOptions {
+  extend CalculatorOptions {
+    optional FeatureDetectorCalculatorOptions ext = 278741680;
+  }
+
+  // Set to true if output patches, otherwise only output cv::KeyPoint
+  optional bool output_patch = 1;
+
+  // The max number of detected features.
+  optional int32 max_features = 2 [default = 200];
+
+  // The number of pyramid levels.
+  optional int32 pyramid_level = 3 [default = 4];
+
+  // Pyramid decimation ratio.
+  optional float scale_factor = 4 [default = 1.2];
+}

mediapipe/calculators/image/image_cropping_calculator.cc

@@ -219,8 +219,10 @@ REGISTER_CALCULATOR(ImageCroppingCalculator);
   const auto& input_img = cc->Inputs().Tag(kImageTag).Get<ImageFrame>();
   cv::Mat input_mat = formats::MatView(&input_img);
 
-  auto [target_width, target_height, rect_center_x, rect_center_y, rotation] =
-      GetCropSpecs(cc, input_img.Width(), input_img.Height());
+  RectSpec specs = GetCropSpecs(cc, input_img.Width(), input_img.Height());
+  int target_width = specs.width, target_height = specs.height,
+      rect_center_x = specs.center_x, rect_center_y = specs.center_y;
+  float rotation = specs.rotation;
 
   // Get border mode and value for OpenCV.
   int border_mode;
@@ -403,8 +405,10 @@ void ImageCroppingCalculator::GetOutputDimensions(CalculatorContext* cc,
                                                   int src_width, int src_height,
                                                   int* dst_width,
                                                   int* dst_height) {
-  auto [crop_width, crop_height, x_center, y_center, rotation] =
-      GetCropSpecs(cc, src_width, src_height);
+  RectSpec specs = GetCropSpecs(cc, src_width, src_height);
+  int crop_width = specs.width, crop_height = specs.height,
+      x_center = specs.center_x, y_center = specs.center_y;
+  float rotation = specs.rotation;
 
   const float half_width = crop_width / 2.0f;
   const float half_height = crop_height / 2.0f;

mediapipe/calculators/image/opencv_encoded_image_to_image_frame_calculator.cc

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include "mediapipe/calculators/image/opencv_encoded_image_to_image_frame_calculator.pb.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/formats/image_frame_opencv.h"
 #include "mediapipe/framework/port/opencv_imgcodecs_inc.h"
@@ -34,7 +35,11 @@ namespace mediapipe {
 class OpenCvEncodedImageToImageFrameCalculator : public CalculatorBase {
  public:
   static ::mediapipe::Status GetContract(CalculatorContract* cc);
+  ::mediapipe::Status Open(CalculatorContext* cc) override;
   ::mediapipe::Status Process(CalculatorContext* cc) override;
+
+ private:
+  mediapipe::OpenCvEncodedImageToImageFrameCalculatorOptions options_;
 };
 
 ::mediapipe::Status OpenCvEncodedImageToImageFrameCalculator::GetContract(
@@ -44,13 +49,29 @@ class OpenCvEncodedImageToImageFrameCalculator : public CalculatorBase {
   return ::mediapipe::OkStatus();
 }
 
+::mediapipe::Status OpenCvEncodedImageToImageFrameCalculator::Open(
+    CalculatorContext* cc) {
+  options_ =
+      cc->Options<mediapipe::OpenCvEncodedImageToImageFrameCalculatorOptions>();
+  return ::mediapipe::OkStatus();
+}
+
 ::mediapipe::Status OpenCvEncodedImageToImageFrameCalculator::Process(
     CalculatorContext* cc) {
   const std::string& contents = cc->Inputs().Index(0).Get<std::string>();
   const std::vector<char> contents_vector(contents.begin(), contents.end());
-  cv::Mat decoded_mat =
-      cv::imdecode(contents_vector, -1 /* return the loaded image as-is */);
-
+  cv::Mat decoded_mat;
+  if (options_.apply_orientation_from_exif_data()) {
+    // We want to respect the orientation from the EXIF data, which
+    // IMREAD_UNCHANGED ignores, but otherwise we want to be as permissive as
+    // possible with our reading flags. Therefore, we use IMREAD_ANYCOLOR and
+    // IMREAD_ANYDEPTH.
+    decoded_mat = cv::imdecode(contents_vector,
+                               cv::IMREAD_ANYCOLOR | cv::IMREAD_ANYDEPTH);
+  } else {
+    // Return the loaded image as-is
+    decoded_mat = cv::imdecode(contents_vector, cv::IMREAD_UNCHANGED);
+  }
   ImageFormat::Format image_format = ImageFormat::UNKNOWN;
   cv::Mat output_mat;
   switch (decoded_mat.channels()) {
@@ -70,7 +91,8 @@ class OpenCvEncodedImageToImageFrameCalculator : public CalculatorBase {
              << "Unsupported number of channels: " << decoded_mat.channels();
   }
   std::unique_ptr<ImageFrame> output_frame = absl::make_unique<ImageFrame>(
-      image_format, decoded_mat.size().width, decoded_mat.size().height);
+      image_format, decoded_mat.size().width, decoded_mat.size().height,
+      ImageFrame::kGlDefaultAlignmentBoundary);
   output_mat.copyTo(formats::MatView(output_frame.get()));
   cc->Outputs().Index(0).Add(output_frame.release(), cc->InputTimestamp());
   return ::mediapipe::OkStatus();

mediapipe/calculators/image/opencv_encoded_image_to_image_frame_calculator.proto

@@ -0,0 +1,30 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+
+message OpenCvEncodedImageToImageFrameCalculatorOptions {
+  extend CalculatorOptions {
+    optional OpenCvEncodedImageToImageFrameCalculatorOptions ext = 303447308;
+  }
+
+  // If set, we will attempt to automatically apply the orientation specified by
+  // the image's EXIF data when loading the image. Otherwise, the image data
+  // will be loaded as-is.
+  optional bool apply_orientation_from_exif_data = 1 [default = false];
+}

mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator.cc

@@ -51,10 +51,11 @@ static constexpr char kStringSavedModelPath[] = "STRING_SAVED_MODEL_PATH";
 #endif
 }
 
-// If options.convert_signature_to_tags() will convert letters to uppercase
-// and replace /'s with _'s. If set, this enables the standard SavedModel
-// classification, regression, and prediction signatures to be used as
-// uppercase INPUTS and OUTPUTS tags for streams.
+// If options.convert_signature_to_tags() is set, will convert letters to
+// uppercase and replace /'s and -'s with _'s. This enables the standard
+// SavedModel classification, regression, and prediction signatures to be used
+// as uppercase INPUTS and OUTPUTS tags for streams and supports other common
+// patterns.
 const std::string MaybeConvertSignatureToTag(
     const std::string& name,
     const TensorFlowSessionFromSavedModelCalculatorOptions& options) {
@@ -64,6 +65,7 @@ const std::string MaybeConvertSignatureToTag(
     std::transform(name.begin(), name.end(), output.begin(),
                    [](unsigned char c) { return std::toupper(c); });
     output = absl::StrReplaceAll(output, {{"/", "_"}});
+    output = absl::StrReplaceAll(output, {{"-", "_"}});
     return output;
   } else {
     return name;

mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator.proto

@@ -32,8 +32,8 @@ message TensorFlowSessionFromSavedModelCalculatorOptions {
   // The name of the generic signature to load into the mapping from tags to
   // tensor names.
   optional string signature_name = 2 [default = "serving_default"];
-  // Whether to convert the signature keys to uppercase and switch /'s to
-  // _'s, which enables standard signatures to be used as Tags.
+  // Whether to convert the signature keys to uppercase as well as switch /'s
+  // and -'s to _'s, which enables common signatures to be used as Tags.
   optional bool convert_signature_to_tags = 3 [default = true];
   // If true, saved_model_path can have multiple exported models in
   // subdirectories saved_model_path/%08d and the alphabetically last (i.e.,

mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator.cc

@@ -53,10 +53,11 @@ static constexpr char kStringSavedModelPath[] = "STRING_SAVED_MODEL_PATH";
 #endif
 }
 
-// If options.convert_signature_to_tags() will convert letters to uppercase
-// and replace /'s with _'s. If set, this enables the standard SavedModel
-// classification, regression, and prediction signatures to be used as
-// uppercase INPUTS and OUTPUTS tags for streams.
+// If options.convert_signature_to_tags() is set, will convert letters to
+// uppercase and replace /'s and -'s with _'s. This enables the standard
+// SavedModel classification, regression, and prediction signatures to be used
+// as uppercase INPUTS and OUTPUTS tags for streams and supports other common
+// patterns.
 const std::string MaybeConvertSignatureToTag(
     const std::string& name,
     const TensorFlowSessionFromSavedModelGeneratorOptions& options) {
@@ -66,6 +67,7 @@ const std::string MaybeConvertSignatureToTag(
     std::transform(name.begin(), name.end(), output.begin(),
                    [](unsigned char c) { return std::toupper(c); });
     output = absl::StrReplaceAll(output, {{"/", "_"}});
+    output = absl::StrReplaceAll(output, {{"-", "_"}});
     return output;
   } else {
     return name;

mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator.proto

@@ -32,8 +32,8 @@ message TensorFlowSessionFromSavedModelGeneratorOptions {
   // The name of the generic signature to load into the mapping from tags to
   // tensor names.
   optional string signature_name = 2 [default = "serving_default"];
-  // Whether to convert the signature keys to uppercase and switch /'s to
-  // _'s, which enables standard signatures to be used as Tags.
+  // Whether to convert the signature keys to uppercase as well as switch /'s
+  // and -'s to _'s, which enables common signatures to be used as Tags.
   optional bool convert_signature_to_tags = 3 [default = true];
   // If true, saved_model_path can have multiple exported models in
   // subdirectories saved_model_path/%08d and the alphabetically last (i.e.,

mediapipe/calculators/tflite/BUILD

@@ -451,7 +451,7 @@ cc_library(
         "//mediapipe:android": [
             "//mediapipe/util/android/file/base",
         ],
-        "//mediapipe:apple": [
+        "//mediapipe:ios": [
             "//mediapipe/util/android/file/base",
         ],
         "//mediapipe:macos": [

mediapipe/calculators/tflite/tflite_inference_calculator.cc

@@ -673,7 +673,7 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
     const auto& input_indices = interpreter_->inputs();
     gpu_data_in_.resize(input_indices.size());
     for (int i = 0; i < input_indices.size(); ++i) {
-      const TfLiteTensor* tensor = interpreter_->tensor(input_indices[0]);
+      const TfLiteTensor* tensor = interpreter_->tensor(input_indices[i]);
       gpu_data_in_[i] = absl::make_unique<GPUData>();
       gpu_data_in_[i]->elements = 1;
       for (int d = 0; d < tensor->dims->size; ++d) {

mediapipe/calculators/tflite/tflite_tensors_to_landmarks_calculator.cc

@@ -145,7 +145,7 @@ REGISTER_CALCULATOR(TfLiteTensorsToLandmarksCalculator);
           ? cc->InputSidePackets().Tag("FLIP_HORIZONTALLY").Get<bool>()
           : options_.flip_horizontally();
 
-  flip_horizontally_ =
+  flip_vertically_ =
       cc->InputSidePackets().HasTag("FLIP_VERTICALLY")
           ? cc->InputSidePackets().Tag("FLIP_VERTICALLY").Get<bool>()
           : options_.flip_vertically();

mediapipe/calculators/tflite/util.h

@@ -15,11 +15,11 @@
 #ifndef MEDIAPIPE_CALCULATORS_TFLITE_UTIL_H_
 #define MEDIAPIPE_CALCULATORS_TFLITE_UTIL_H_
 
-#define RET_CHECK_CALL(call)                                     \
-  do {                                                           \
-    const auto status = (call);                                  \
-    if (ABSL_PREDICT_FALSE(!status.ok()))                        \
-      return ::mediapipe::InternalError(status.error_message()); \
+#define RET_CHECK_CALL(call)                               \
+  do {                                                     \
+    const auto status = (call);                            \
+    if (ABSL_PREDICT_FALSE(!status.ok()))                  \
+      return ::mediapipe::InternalError(status.message()); \
   } while (0);
 
 #endif  // MEDIAPIPE_CALCULATORS_TFLITE_UTIL_H_

mediapipe/calculators/util/BUILD

@@ -321,7 +321,7 @@ cc_library(
         "//mediapipe:android": [
             "//mediapipe/util/android/file/base",
         ],
-        "//mediapipe:apple": [
+        "//mediapipe:ios": [
             "//mediapipe/util/android/file/base",
         ],
         "//mediapipe:macos": [
@@ -349,7 +349,7 @@ cc_library(
         "//mediapipe:android": [
             "//mediapipe/util/android/file/base",
         ],
-        "//mediapipe:apple": [
+        "//mediapipe:ios": [
             "//mediapipe/util/android/file/base",
         ],
         "//mediapipe:macos": [
@@ -926,7 +926,7 @@ cc_library(
         "//mediapipe:android": [
             "//mediapipe/util/android/file/base",
         ],
-        "//mediapipe:apple": [
+        "//mediapipe:ios": [
             "//mediapipe/util/android/file/base",
         ],
         "//mediapipe:macos": [
@@ -971,9 +971,9 @@ cc_library(
     visibility = ["//visibility:public"],
     deps = [
         "//mediapipe/framework:calculator_framework",
-        "//mediapipe/framework/port:file_helpers",
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework/port:status",
+        "//mediapipe/util:resource_util",
     ],
     alwayslink = 1,
 )

mediapipe/calculators/util/annotation_overlay_calculator.cc

@@ -55,7 +55,7 @@ size_t RoundUp(size_t n, size_t m) { return ((n + m - 1) / m) * m; }  // NOLINT
 // When using GPU, this color will become transparent when the calculator
 // merges the annotation overlay with the image frame. As a result, drawing in
 // this color is not supported and it should be set to something unlikely used.
-constexpr int kAnnotationBackgroundColor[] = {100, 101, 102};
+constexpr uchar kAnnotationBackgroundColor = 2;  // Grayscale value.
 }  // namespace
 
 // A calculator for rendering data on images.
@@ -491,11 +491,9 @@ REGISTER_CALCULATOR(AnnotationOverlayCalculator);
     if (format != mediapipe::ImageFormat::SRGBA &&
         format != mediapipe::ImageFormat::SRGB)
       RET_CHECK_FAIL() << "Unsupported GPU input format: " << format;
-
-    image_mat = absl::make_unique<cv::Mat>(
-        height_, width_, CV_8UC3,
-        cv::Scalar(kAnnotationBackgroundColor[0], kAnnotationBackgroundColor[1],
-                   kAnnotationBackgroundColor[2]));
+    image_mat = absl::make_unique<cv::Mat>(height_, width_, CV_8UC3);
+    memset(image_mat->data, kAnnotationBackgroundColor,
+           height_ * width_ * image_mat->elemSize());
   } else {
     image_mat = absl::make_unique<cv::Mat>(
         options_.canvas_height_px(), options_.canvas_width_px(), CV_8UC3,
@@ -617,9 +615,9 @@ REGISTER_CALCULATOR(AnnotationOverlayCalculator);
   glUniform1i(glGetUniformLocation(program_, "input_frame"), 1);
   glUniform1i(glGetUniformLocation(program_, "overlay"), 2);
   glUniform3f(glGetUniformLocation(program_, "transparent_color"),
-              kAnnotationBackgroundColor[0] / 255.0,
-              kAnnotationBackgroundColor[1] / 255.0,
-              kAnnotationBackgroundColor[2] / 255.0);
+              kAnnotationBackgroundColor / 255.0,
+              kAnnotationBackgroundColor / 255.0,
+              kAnnotationBackgroundColor / 255.0);
 
   // Init texture for opencv rendered frame.
   const auto& input_frame =

mediapipe/calculators/util/labels_to_render_data_calculator.cc

@@ -128,16 +128,19 @@ REGISTER_CALCULATOR(LabelsToRenderDataCalculator);
   } else {
     const std::vector<std::string>& label_vector =
         cc->Inputs().Tag("LABELS").Get<std::vector<std::string>>();
-    std::vector<float> score_vector;
-    if (cc->Inputs().HasTag("SCORES")) {
-      score_vector = cc->Inputs().Tag("SCORES").Get<std::vector<float>>();
-    }
-    CHECK_EQ(label_vector.size(), score_vector.size());
     labels.resize(label_vector.size());
-    scores.resize(label_vector.size());
     for (int i = 0; i < label_vector.size(); ++i) {
       labels[i] = label_vector[i];
-      scores[i] = score_vector[i];
+    }
+
+    if (cc->Inputs().HasTag("SCORES")) {
+      std::vector<float> score_vector =
+          cc->Inputs().Tag("SCORES").Get<std::vector<float>>();
+      CHECK_EQ(label_vector.size(), score_vector.size());
+      scores.resize(label_vector.size());
+      for (int i = 0; i < label_vector.size(); ++i) {
+        scores[i] = score_vector[i];
+      }
     }
   }
 

mediapipe/calculators/util/local_file_contents_calculator.cc

@@ -16,34 +16,80 @@
 #include <string>
 
 #include "mediapipe/framework/calculator_framework.h"
-#include "mediapipe/framework/port/file_helpers.h"
 #include "mediapipe/framework/port/status.h"
+#include "mediapipe/util/resource_util.h"
 
 namespace mediapipe {
+
+namespace {
+
+constexpr char kFilePathTag[] = "FILE_PATH";
+constexpr char kContentsTag[] = "CONTENTS";
+
+}  // namespace
+
 // The calculator takes the path to the local file as an input side packet and
 // outputs the contents of that file.
 //
+// NOTE: file loading can be batched by providing multiple input/output side
+// packets.
+//
 // Example config:
 // node {
 //   calculator: "LocalFileContentsCalculator"
 //   input_side_packet: "FILE_PATH:file_path"
 //   output_side_packet: "CONTENTS:contents"
 // }
+//
+// node {
+//   calculator: "LocalFileContentsCalculator"
+//   input_side_packet: "FILE_PATH:0:file_path1"
+//   input_side_packet: "FILE_PATH:1:file_path2"
+//   ...
+//   output_side_packet: "CONTENTS:0:contents1"
+//   output_side_packet: "CONTENTS:1:contents2"
+//   ...
+// }
 class LocalFileContentsCalculator : public CalculatorBase {
  public:
   static ::mediapipe::Status GetContract(CalculatorContract* cc) {
-    cc->InputSidePackets().Tag("FILE_PATH").Set<std::string>();
-    cc->OutputSidePackets().Tag("CONTENTS").Set<std::string>();
+    RET_CHECK(cc->InputSidePackets().HasTag(kFilePathTag))
+        << "Missing PATH input side packet(s)";
+    RET_CHECK(cc->OutputSidePackets().HasTag(kContentsTag))
+        << "Missing CONTENTS output side packet(s)";
+
+    RET_CHECK_EQ(cc->InputSidePackets().NumEntries(kFilePathTag),
+                 cc->OutputSidePackets().NumEntries(kContentsTag))
+        << "Same number of input streams and output streams is required.";
+
+    for (CollectionItemId id = cc->InputSidePackets().BeginId(kFilePathTag);
+         id != cc->InputSidePackets().EndId(kFilePathTag); ++id) {
+      cc->InputSidePackets().Get(id).Set<std::string>();
+    }
+
+    for (CollectionItemId id = cc->OutputSidePackets().BeginId(kContentsTag);
+         id != cc->OutputSidePackets().EndId(kContentsTag); ++id) {
+      cc->OutputSidePackets().Get(id).Set<std::string>();
+    }
+
     return ::mediapipe::OkStatus();
   }
 
   ::mediapipe::Status Open(CalculatorContext* cc) override {
-    std::string contents;
-    MP_RETURN_IF_ERROR(mediapipe::file::GetContents(
-        cc->InputSidePackets().Tag("FILE_PATH").Get<std::string>(), &contents));
-    cc->OutputSidePackets()
-        .Tag("CONTENTS")
-        .Set(MakePacket<std::string>(std::move(contents)));
+    CollectionItemId input_id = cc->InputSidePackets().BeginId(kFilePathTag);
+    CollectionItemId output_id = cc->OutputSidePackets().BeginId(kContentsTag);
+    // Number of inputs and outpus is the same according to the contract.
+    for (; input_id != cc->InputSidePackets().EndId(kFilePathTag);
+         ++input_id, ++output_id) {
+      std::string file_path =
+          cc->InputSidePackets().Get(input_id).Get<std::string>();
+      ASSIGN_OR_RETURN(file_path, PathToResourceAsFile(file_path));
+
+      std::string contents;
+      MP_RETURN_IF_ERROR(GetResourceContents(file_path, &contents));
+      cc->OutputSidePackets().Get(output_id).Set(
+          MakePacket<std::string>(std::move(contents)));
+    }
     return ::mediapipe::OkStatus();
   }
 

mediapipe/calculators/util/timed_box_list_to_render_data_calculator.cc

@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include <algorithm>
+
 #include "absl/memory/memory.h"
 #include "absl/strings/str_cat.h"
 #include "absl/strings/str_join.h"
@@ -76,14 +78,16 @@ void AddTimedBoxProtoToRenderData(
     RenderAnnotation::Text* text = label_annotation->mutable_text();
     text->set_display_text(box_proto.label());
     text->set_normalized(true);
-    constexpr float text_left_start = 0.3f;
+    constexpr float text_left_start = 0.2f;
     text->set_left((1.0f - text_left_start) * box_proto.left() +
                    text_left_start * box_proto.right());
     constexpr float text_baseline = 0.6f;
     text->set_baseline(text_baseline * box_proto.bottom() +
                        (1.0f - text_baseline) * box_proto.top());
-    constexpr float text_height = 0.2f;
-    text->set_font_height((box_proto.bottom() - box_proto.top()) * text_height);
+    constexpr float text_height = 0.1f;
+    text->set_font_height(std::min(box_proto.bottom() - box_proto.top(),
+                                   box_proto.right() - box_proto.left()) *
+                          text_height);
   }
 }
 

mediapipe/calculators/video/BUILD

@@ -65,6 +65,26 @@ proto_library(
     ],
 )
 
+proto_library(
+    name = "tracked_detection_manager_calculator_proto",
+    srcs = ["tracked_detection_manager_calculator.proto"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_proto",
+        "//mediapipe/util/tracking:tracked_detection_manager_config_proto",
+    ],
+)
+
+proto_library(
+    name = "box_detector_calculator_proto",
+    srcs = ["box_detector_calculator.proto"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_proto",
+        "//mediapipe/util/tracking:box_detector_proto",
+    ],
+)
+
 proto_library(
     name = "video_pre_stream_calculator_proto",
     srcs = ["video_pre_stream_calculator.proto"],
@@ -107,6 +127,28 @@ mediapipe_cc_proto_library(
     deps = [":box_tracker_calculator_proto"],
 )
 
+mediapipe_cc_proto_library(
+    name = "tracked_detection_manager_calculator_cc_proto",
+    srcs = ["tracked_detection_manager_calculator.proto"],
+    cc_deps = [
+        "//mediapipe/framework:calculator_cc_proto",
+        "//mediapipe/util/tracking:tracked_detection_manager_config_cc_proto",
+    ],
+    visibility = ["//visibility:public"],
+    deps = [":tracked_detection_manager_calculator_proto"],
+)
+
+mediapipe_cc_proto_library(
+    name = "box_detector_calculator_cc_proto",
+    srcs = ["box_detector_calculator.proto"],
+    cc_deps = [
+        "//mediapipe/framework:calculator_cc_proto",
+        "//mediapipe/util/tracking:box_detector_cc_proto",
+    ],
+    visibility = ["//visibility:public"],
+    deps = [":box_detector_calculator_proto"],
+)
+
 mediapipe_cc_proto_library(
     name = "video_pre_stream_calculator_cc_proto",
     srcs = ["video_pre_stream_calculator.proto"],
@@ -279,11 +321,54 @@ cc_library(
     alwayslink = 1,
 )
 
+cc_library(
+    name = "box_detector_calculator",
+    srcs = ["box_detector_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":box_detector_calculator_cc_proto",
+        "@com_google_absl//absl/memory",
+        "@com_google_absl//absl/strings",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:image_frame",
+        "//mediapipe/framework/formats:image_frame_opencv",
+        "//mediapipe/framework/formats:video_stream_header",  # fixdeps: keep -- required for exobazel build.
+        "//mediapipe/framework/port:integral_types",
+        "//mediapipe/framework/port:logging",
+        "//mediapipe/framework/port:opencv_core",
+        "//mediapipe/framework/port:opencv_features2d",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/util:resource_util",
+        "//mediapipe/util/tracking",
+        "//mediapipe/util/tracking:box_detector",
+        "//mediapipe/util/tracking:box_tracker",
+        "//mediapipe/util/tracking:box_tracker_cc_proto",
+        "//mediapipe/util/tracking:flow_packager_cc_proto",
+        "//mediapipe/util/tracking:tracking_visualization_utilities",
+    ] + select({
+        "//mediapipe:android": [
+            "//mediapipe/util/android/file/base",
+        ],
+        "//mediapipe:ios": [
+            "//mediapipe/util/android/file/base",
+        ],
+        "//mediapipe:macos": [
+            "//mediapipe/framework/port:file_helpers",
+        ],
+        "//conditions:default": [
+            "//mediapipe/framework/port:file_helpers",
+        ],
+    }),
+    alwayslink = 1,
+)
+
 cc_library(
     name = "tracked_detection_manager_calculator",
     srcs = ["tracked_detection_manager_calculator.cc"],
     visibility = ["//visibility:public"],
     deps = [
+        ":tracked_detection_manager_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework/formats:detection_cc_proto",
         "//mediapipe/framework/formats:location_data_cc_proto",

mediapipe/calculators/video/box_detector_calculator.cc

@@ -0,0 +1,393 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stdio.h>
+
+#include <memory>
+#include <unordered_set>
+
+#include "absl/memory/memory.h"
+#include "absl/strings/numbers.h"
+#include "mediapipe/calculators/video/box_detector_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/image_frame.h"
+#include "mediapipe/framework/formats/image_frame_opencv.h"
+#include "mediapipe/framework/formats/video_stream_header.h"
+#include "mediapipe/framework/port/integral_types.h"
+#include "mediapipe/framework/port/logging.h"
+#include "mediapipe/framework/port/opencv_core_inc.h"
+#include "mediapipe/framework/port/opencv_features2d_inc.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/util/resource_util.h"
+#include "mediapipe/util/tracking/box_detector.h"
+#include "mediapipe/util/tracking/box_tracker.h"
+#include "mediapipe/util/tracking/box_tracker.pb.h"
+#include "mediapipe/util/tracking/flow_packager.pb.h"
+#include "mediapipe/util/tracking/tracking.h"
+#include "mediapipe/util/tracking/tracking_visualization_utilities.h"
+
+#if defined(MEDIAPIPE_MOBILE)
+#include "mediapipe/util/android/file/base/file.h"
+#include "mediapipe/util/android/file/base/helpers.h"
+#else
+#include "mediapipe/framework/port/file_helpers.h"
+#endif
+
+namespace mediapipe {
+
+// A calculator to detect reappeared box positions from single frame.
+//
+// Input stream:
+//   TRACKING: Input tracking data (proto TrackingData) containing features and
+//             descriptors.
+//   VIDEO:    Optional input video stream tracked boxes are rendered over
+//             (Required if VIZ is specified).
+//   FEATURES: Input feature points (std::vector<cv::KeyPoint>) in the original
+//             pixel space.
+//   DESCRIPTORS: Input feature descriptors (std::vector<float>). Actual feature
+//             dimension needs to be specified in detector_options.
+//   IMAGE_SIZE: Input image dimension.
+//   TRACKED_BOXES : input box tracking result (proto TimedBoxProtoList) from
+//             BoxTrackerCalculator.
+//   ADD_INDEX: Optional std::string containing binary format proto of type
+//             BoxDetectorIndex. Used for adding target index to the detector
+//             search index during runtime.
+//   CANCEL_OBJECT_ID: Optional id of box to be removed. This is recommended
+//             to be used with SyncSetInputStreamHandler.
+//   REACQ_SWITCH: Optional bool for swithcing on and off reacquisition
+//             functionality. User should initialize a graph with box detector
+//             calculator and be able to switch it on and off in runtime.
+//
+// Output streams:
+//   VIZ:   Optional output video stream with rendered box positions
+//          (requires VIDEO to be present)
+//   BOXES: Optional output stream of type TimedBoxProtoList for each lost box.
+//
+// Imput side packets:
+//   INDEX_PROTO_STRING: Optional std::string containing binary format proto of
+//   type
+//                       BoxDetectorIndex. Used for initializing box_detector
+//                       with predefined template images.
+//   FRAME_ALIGNMENT:    Optional integer to indicate alignment_boundary for
+//                       outputing ImageFrame in "VIZ" stream.
+//                       Set to ImageFrame::kDefaultAlignmentBoundary for
+//                       offline pipeline to be compatible with FFmpeg.
+//                       Set to ImageFrame::kGlDefaultAlignmentBoundary for Apps
+//                       to be compatible with GL renderer.
+//   OUTPUT_INDEX_FILENAME: File path to the output index file.
+
+class BoxDetectorCalculator : public CalculatorBase {
+ public:
+  ~BoxDetectorCalculator() override = default;
+
+  static ::mediapipe::Status GetContract(CalculatorContract* cc);
+
+  ::mediapipe::Status Open(CalculatorContext* cc) override;
+  ::mediapipe::Status Process(CalculatorContext* cc) override;
+  ::mediapipe::Status Close(CalculatorContext* cc) override;
+
+ private:
+  BoxDetectorCalculatorOptions options_;
+  std::unique_ptr<BoxDetectorInterface> box_detector_;
+  bool detector_switch_ = true;
+  uint32 frame_alignment_ = ImageFrame::kDefaultAlignmentBoundary;
+  bool write_index_ = false;
+  int box_id_ = 0;
+};
+
+REGISTER_CALCULATOR(BoxDetectorCalculator);
+
+::mediapipe::Status BoxDetectorCalculator::GetContract(CalculatorContract* cc) {
+  if (cc->Inputs().HasTag("TRACKING")) {
+    cc->Inputs().Tag("TRACKING").Set<TrackingData>();
+  }
+
+  if (cc->Inputs().HasTag("TRACKED_BOXES")) {
+    cc->Inputs().Tag("TRACKED_BOXES").Set<TimedBoxProtoList>();
+  }
+
+  if (cc->Inputs().HasTag("VIDEO")) {
+    cc->Inputs().Tag("VIDEO").Set<ImageFrame>();
+  }
+
+  if (cc->Inputs().HasTag("FEATURES")) {
+    RET_CHECK(cc->Inputs().HasTag("DESCRIPTORS"))
+        << "FEATURES and DESCRIPTORS need to be specified together.";
+    cc->Inputs().Tag("FEATURES").Set<std::vector<cv::KeyPoint>>();
+  }
+
+  if (cc->Inputs().HasTag("DESCRIPTORS")) {
+    RET_CHECK(cc->Inputs().HasTag("FEATURES"))
+        << "FEATURES and DESCRIPTORS need to be specified together.";
+    cc->Inputs().Tag("DESCRIPTORS").Set<std::vector<float>>();
+  }
+
+  if (cc->Inputs().HasTag("IMAGE_SIZE")) {
+    cc->Inputs().Tag("IMAGE_SIZE").Set<std::pair<int, int>>();
+  }
+
+  if (cc->Inputs().HasTag("ADD_INDEX")) {
+    cc->Inputs().Tag("ADD_INDEX").Set<std::string>();
+  }
+
+  if (cc->Inputs().HasTag("CANCEL_OBJECT_ID")) {
+    cc->Inputs().Tag("CANCEL_OBJECT_ID").Set<int>();
+  }
+
+  if (cc->Inputs().HasTag("REACQ_SWITCH")) {
+    cc->Inputs().Tag("REACQ_SWITCH").Set<bool>();
+  }
+
+  if (cc->Outputs().HasTag("BOXES")) {
+    cc->Outputs().Tag("BOXES").Set<TimedBoxProtoList>();
+  }
+
+  if (cc->Outputs().HasTag("VIZ")) {
+    RET_CHECK(cc->Inputs().HasTag("VIDEO"))
+        << "Output stream VIZ requires VIDEO to be present.";
+    cc->Outputs().Tag("VIZ").Set<ImageFrame>();
+  }
+
+  if (cc->InputSidePackets().HasTag("INDEX_PROTO_STRING")) {
+    cc->InputSidePackets().Tag("INDEX_PROTO_STRING").Set<std::string>();
+  }
+
+  if (cc->InputSidePackets().HasTag("OUTPUT_INDEX_FILENAME")) {
+    cc->InputSidePackets().Tag("OUTPUT_INDEX_FILENAME").Set<std::string>();
+  }
+
+  if (cc->InputSidePackets().HasTag("FRAME_ALIGNMENT")) {
+    cc->InputSidePackets().Tag("FRAME_ALIGNMENT").Set<int>();
+  }
+
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status BoxDetectorCalculator::Open(CalculatorContext* cc) {
+  options_ = cc->Options<BoxDetectorCalculatorOptions>();
+  box_detector_ = BoxDetectorInterface::Create(options_.detector_options());
+
+  if (cc->InputSidePackets().HasTag("INDEX_PROTO_STRING")) {
+    BoxDetectorIndex predefined_index;
+    if (!predefined_index.ParseFromString(cc->InputSidePackets()
+                                              .Tag("INDEX_PROTO_STRING")
+                                              .Get<std::string>())) {
+      LOG(FATAL) << "failed to parse BoxDetectorIndex from INDEX_PROTO_STRING";
+    }
+    box_detector_->AddBoxDetectorIndex(predefined_index);
+  }
+
+  for (const auto& filename : options_.index_proto_filename()) {
+    std::string string_path;
+    ASSIGN_OR_RETURN(string_path, PathToResourceAsFile(filename));
+    std::string index_string;
+    MP_RETURN_IF_ERROR(file::GetContents(string_path, &index_string));
+    BoxDetectorIndex predefined_index;
+    if (!predefined_index.ParseFromString(index_string)) {
+      LOG(FATAL)
+          << "failed to parse BoxDetectorIndex from index_proto_filename";
+    }
+    box_detector_->AddBoxDetectorIndex(predefined_index);
+  }
+
+  if (cc->InputSidePackets().HasTag("OUTPUT_INDEX_FILENAME")) {
+    write_index_ = true;
+  }
+
+  if (cc->InputSidePackets().HasTag("FRAME_ALIGNMENT")) {
+    frame_alignment_ = cc->InputSidePackets().Tag("FRAME_ALIGNMENT").Get<int>();
+  }
+
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status BoxDetectorCalculator::Process(CalculatorContext* cc) {
+  const Timestamp timestamp = cc->InputTimestamp();
+  const int64 timestamp_msec = timestamp.Value() / 1000;
+
+  InputStream* cancel_object_id_stream =
+      cc->Inputs().HasTag("CANCEL_OBJECT_ID")
+          ? &(cc->Inputs().Tag("CANCEL_OBJECT_ID"))
+          : nullptr;
+  if (cancel_object_id_stream && !cancel_object_id_stream->IsEmpty()) {
+    const int cancel_object_id = cancel_object_id_stream->Get<int>();
+    box_detector_->CancelBoxDetection(cancel_object_id);
+  }
+
+  InputStream* add_index_stream = cc->Inputs().HasTag("ADD_INDEX")
+                                      ? &(cc->Inputs().Tag("ADD_INDEX"))
+                                      : nullptr;
+  if (add_index_stream && !add_index_stream->IsEmpty()) {
+    BoxDetectorIndex predefined_index;
+    if (!predefined_index.ParseFromString(
+            add_index_stream->Get<std::string>())) {
+      LOG(FATAL) << "failed to parse BoxDetectorIndex from ADD_INDEX";
+    }
+    box_detector_->AddBoxDetectorIndex(predefined_index);
+  }
+
+  InputStream* reacq_switch_stream = cc->Inputs().HasTag("REACQ_SWITCH")
+                                         ? &(cc->Inputs().Tag("REACQ_SWITCH"))
+                                         : nullptr;
+  if (reacq_switch_stream && !reacq_switch_stream->IsEmpty()) {
+    detector_switch_ = reacq_switch_stream->Get<bool>();
+  }
+
+  if (!detector_switch_) {
+    return ::mediapipe::OkStatus();
+  }
+
+  InputStream* track_stream = cc->Inputs().HasTag("TRACKING")
+                                  ? &(cc->Inputs().Tag("TRACKING"))
+                                  : nullptr;
+  InputStream* video_stream =
+      cc->Inputs().HasTag("VIDEO") ? &(cc->Inputs().Tag("VIDEO")) : nullptr;
+  InputStream* feature_stream = cc->Inputs().HasTag("FEATURES")
+                                    ? &(cc->Inputs().Tag("FEATURES"))
+                                    : nullptr;
+  InputStream* descriptor_stream = cc->Inputs().HasTag("DESCRIPTORS")
+                                       ? &(cc->Inputs().Tag("DESCRIPTORS"))
+                                       : nullptr;
+
+  CHECK(track_stream != nullptr || video_stream != nullptr ||
+        (feature_stream != nullptr && descriptor_stream != nullptr))
+      << "One and only one of {tracking_data, input image frame, "
+         "feature/descriptor} need to be valid.";
+
+  InputStream* tracked_boxes_stream = cc->Inputs().HasTag("TRACKED_BOXES")
+                                          ? &(cc->Inputs().Tag("TRACKED_BOXES"))
+                                          : nullptr;
+  std::unique_ptr<TimedBoxProtoList> detected_boxes(new TimedBoxProtoList());
+
+  if (track_stream != nullptr) {
+    // Detect from tracking data
+    if (track_stream->IsEmpty()) {
+      return ::mediapipe::OkStatus();
+    }
+
+    const TrackingData& tracking_data = track_stream->Get<TrackingData>();
+
+    CHECK(tracked_boxes_stream != nullptr) << "tracked_boxes needed.";
+
+    const TimedBoxProtoList tracked_boxes =
+        tracked_boxes_stream->Get<TimedBoxProtoList>();
+
+    box_detector_->DetectAndAddBox(tracking_data, tracked_boxes, timestamp_msec,
+                                   detected_boxes.get());
+  } else if (video_stream != nullptr) {
+    // Detect from input frame
+    if (video_stream->IsEmpty()) {
+      return ::mediapipe::OkStatus();
+    }
+
+    TimedBoxProtoList tracked_boxes;
+    if (tracked_boxes_stream != nullptr && !tracked_boxes_stream->IsEmpty()) {
+      tracked_boxes = tracked_boxes_stream->Get<TimedBoxProtoList>();
+    }
+
+    // Just directly pass along the image frame data as-is for detection; we
+    // don't need to worry about conforming to a specific alignment here.
+    const cv::Mat input_view =
+        formats::MatView(&video_stream->Get<ImageFrame>());
+    box_detector_->DetectAndAddBox(input_view, tracked_boxes, timestamp_msec,
+                                   detected_boxes.get());
+  } else {
+    if (feature_stream->IsEmpty() || descriptor_stream->IsEmpty()) {
+      return ::mediapipe::OkStatus();
+    }
+
+    const auto& image_size =
+        cc->Inputs().Tag("IMAGE_SIZE").Get<std::pair<int, int>>();
+    float inv_scale = 1.0f / std::max(image_size.first, image_size.second);
+
+    TimedBoxProtoList tracked_boxes;
+    if (tracked_boxes_stream != nullptr && !tracked_boxes_stream->IsEmpty()) {
+      tracked_boxes = tracked_boxes_stream->Get<TimedBoxProtoList>();
+    } else if (write_index_) {
+      auto* box_ptr = tracked_boxes.add_box();
+      box_ptr->set_id(box_id_);
+      box_ptr->set_reacquisition(true);
+      box_ptr->set_aspect_ratio((float)image_size.first /
+                                (float)image_size.second);
+
+      box_ptr->mutable_quad()->add_vertices(0);
+      box_ptr->mutable_quad()->add_vertices(0);
+
+      box_ptr->mutable_quad()->add_vertices(0);
+      box_ptr->mutable_quad()->add_vertices(1);
+
+      box_ptr->mutable_quad()->add_vertices(1);
+      box_ptr->mutable_quad()->add_vertices(1);
+
+      box_ptr->mutable_quad()->add_vertices(1);
+      box_ptr->mutable_quad()->add_vertices(0);
+
+      ++box_id_;
+    }
+
+    const auto& features = feature_stream->Get<std::vector<cv::KeyPoint>>();
+    const int feature_size = features.size();
+    std::vector<Vector2_f> features_vec(feature_size);
+
+    const auto& descriptors = descriptor_stream->Get<std::vector<float>>();
+    const int dims = options_.detector_options().descriptor_dims();
+    CHECK_GE(descriptors.size(), feature_size * dims);
+    cv::Mat descriptors_mat(feature_size, dims, CV_32F);
+    for (int j = 0; j < feature_size; ++j) {
+      features_vec[j].Set(features[j].pt.x * inv_scale,
+                          features[j].pt.y * inv_scale);
+      for (int i = 0; i < dims; ++i) {
+        descriptors_mat.at<float>(j, i) = descriptors[j * dims + i];
+      }
+    }
+
+    box_detector_->DetectAndAddBoxFromFeatures(
+        features_vec, descriptors_mat, tracked_boxes, timestamp_msec,
+        image_size.first * inv_scale, image_size.second * inv_scale,
+        detected_boxes.get());
+  }
+
+  if (cc->Outputs().HasTag("VIZ")) {
+    cv::Mat viz_view;
+    std::unique_ptr<ImageFrame> viz_frame;
+    if (video_stream != nullptr && !video_stream->IsEmpty()) {
+      viz_frame = absl::make_unique<ImageFrame>();
+      viz_frame->CopyFrom(video_stream->Get<ImageFrame>(), frame_alignment_);
+      viz_view = formats::MatView(viz_frame.get());
+    }
+    for (const auto& box : detected_boxes->box()) {
+      RenderBox(box, &viz_view);
+    }
+    cc->Outputs().Tag("VIZ").Add(viz_frame.release(), timestamp);
+  }
+
+  if (cc->Outputs().HasTag("BOXES")) {
+    cc->Outputs().Tag("BOXES").Add(detected_boxes.release(), timestamp);
+  }
+
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status BoxDetectorCalculator::Close(CalculatorContext* cc) {
+  if (write_index_) {
+    BoxDetectorIndex index = box_detector_->ObtainBoxDetectorIndex();
+    MEDIAPIPE_CHECK_OK(mediapipe::file::SetContents(
+        cc->InputSidePackets().Tag("OUTPUT_INDEX_FILENAME").Get<std::string>(),
+        index.SerializeAsString()));
+  }
+  return ::mediapipe::OkStatus();
+}
+
+}  // namespace mediapipe

mediapipe/calculators/video/box_detector_calculator.proto

@@ -0,0 +1,31 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+import "mediapipe/util/tracking/box_detector.proto";
+
+message BoxDetectorCalculatorOptions {
+  extend CalculatorOptions {
+    optional BoxDetectorCalculatorOptions ext = 289746530;
+  }
+
+  optional BoxDetectorOptions detector_options = 1;
+
+  // File path to the template index files.
+  repeated string index_proto_filename = 2;
+}

mediapipe/calculators/video/tracked_detection_manager_calculator.cc

@@ -18,6 +18,7 @@
 #include <vector>
 
 #include "absl/container/node_hash_map.h"
+#include "mediapipe/calculators/video/tracked_detection_manager_calculator.pb.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/formats/detection.pb.h"
 #include "mediapipe/framework/formats/location_data.pb.h"
@@ -139,6 +140,7 @@ Detection GetAxisAlignedDetectionFromTrackedDetection(
 class TrackedDetectionManagerCalculator : public CalculatorBase {
  public:
   static ::mediapipe::Status GetContract(CalculatorContract* cc);
+  ::mediapipe::Status Open(CalculatorContext* cc) override;
 
   ::mediapipe::Status Process(CalculatorContext* cc) override;
 
@@ -184,6 +186,15 @@ REGISTER_CALCULATOR(TrackedDetectionManagerCalculator);
   return ::mediapipe::OkStatus();
 }
 
+::mediapipe::Status TrackedDetectionManagerCalculator::Open(
+    CalculatorContext* cc) {
+  mediapipe::TrackedDetectionManagerCalculatorOptions options =
+      cc->Options<mediapipe::TrackedDetectionManagerCalculatorOptions>();
+  tracked_detection_manager_.SetConfig(
+      options.tracked_detection_manager_options());
+  return ::mediapipe::OkStatus();
+}
+
 ::mediapipe::Status TrackedDetectionManagerCalculator::Process(
     CalculatorContext* cc) {
   if (cc->Inputs().HasTag("TRACKING_BOXES")) {

mediapipe/calculators/video/tracked_detection_manager_calculator.proto

@@ -0,0 +1,28 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+import "mediapipe/util/tracking/tracked_detection_manager_config.proto";
+
+message TrackedDetectionManagerCalculatorOptions {
+  extend CalculatorOptions {
+    optional TrackedDetectionManagerCalculatorOptions ext = 301970230;
+  }
+
+  optional TrackedDetectionManagerConfig tracked_detection_manager_options = 1;
+}