Project import generated by Copybara.

GitOrigin-RevId: 5b23708185311ae39a8605b0c2eff721e7b4939f
2020-08-05 20:27:31 -04:00
parent bdfdaef305
commit 2f86a459b6
115 changed files with 5242 additions and 251 deletions
@@ -165,28 +165,53 @@ REGISTER_CALCULATOR(ContentZoomingCalculator);
 }

 namespace {
-::mediapipe::Status UpdateRanges(const SalientRegion& region, float* xmin,
+mediapipe::LocationData::RelativeBoundingBox ShiftDetection(
+    const mediapipe::LocationData::RelativeBoundingBox& relative_bounding_box,
+    const float y_offset_percent, const float x_offset_percent) {
+  auto shifted_bb = relative_bounding_box;
+  shifted_bb.set_ymin(relative_bounding_box.ymin() +
+                      relative_bounding_box.height() * y_offset_percent);
+  shifted_bb.set_xmin(relative_bounding_box.xmin() +
+                      relative_bounding_box.width() * x_offset_percent);
+  return shifted_bb;
+}
+mediapipe::autoflip::RectF ShiftDetection(
+    const mediapipe::autoflip::RectF& relative_bounding_box,
+    const float y_offset_percent, const float x_offset_percent) {
+  auto shifted_bb = relative_bounding_box;
+  shifted_bb.set_y(relative_bounding_box.y() +
+                   relative_bounding_box.height() * y_offset_percent);
+  shifted_bb.set_x(relative_bounding_box.x() +
+                   relative_bounding_box.width() * x_offset_percent);
+  return shifted_bb;
+}
+::mediapipe::Status UpdateRanges(const SalientRegion& region,
+                                 const float shift_vertical,
+                                 const float shift_horizontal, float* xmin,
                                 float* xmax, float* ymin, float* ymax) {
  if (!region.has_location_normalized()) {
    return ::mediapipe::UnknownErrorBuilder(MEDIAPIPE_LOC)
           << "SalientRegion did not have location normalized set.";
  }
-  *xmin = fmin(*xmin, region.location_normalized().x());
-  *xmax = fmax(*xmax, region.location_normalized().x() +
-                          region.location_normalized().width());
-  *ymin = fmin(*ymin, region.location_normalized().y());
-  *ymax = fmax(*ymax, region.location_normalized().y() +
-                          region.location_normalized().height());
+  auto location = ShiftDetection(region.location_normalized(), shift_vertical,
+                                 shift_horizontal);
+  *xmin = fmin(*xmin, location.x());
+  *xmax = fmax(*xmax, location.x() + location.width());
+  *ymin = fmin(*ymin, location.y());
+  *ymax = fmax(*ymax, location.y() + location.height());

  return ::mediapipe::OkStatus();
 }
 ::mediapipe::Status UpdateRanges(const mediapipe::Detection& detection,
-                                 float* xmin, float* xmax, float* ymin,
-                                 float* ymax) {
+                                 const float shift_vertical,
+                                 const float shift_horizontal, float* xmin,
+                                 float* xmax, float* ymin, float* ymax) {
  RET_CHECK(detection.location_data().format() ==
            mediapipe::LocationData::RELATIVE_BOUNDING_BOX)
      << "Face detection input is lacking required relative_bounding_box()";
-  const auto& location = detection.location_data().relative_bounding_box();
+  const auto& location =
+      ShiftDetection(detection.location_data().relative_bounding_box(),
+                     shift_vertical, shift_horizontal);
  *xmin = fmin(*xmin, location.xmin());
  *xmax = fmax(*xmax, location.xmin() + location.width());
  *ymin = fmin(*ymin, location.ymin());
@@ -270,7 +295,9 @@ void MakeStaticFeatures(const int top_border, const int bottom_border,
        continue;
      }
      only_required_found = true;
-      MP_RETURN_IF_ERROR(UpdateRanges(region, &xmin, &xmax, &ymin, &ymax));
+      MP_RETURN_IF_ERROR(UpdateRanges(
+          region, options_.detection_shift_vertical(),
+          options_.detection_shift_horizontal(), &xmin, &xmax, &ymin, &ymax));
    }
  }

@@ -279,7 +306,9 @@ void MakeStaticFeatures(const int top_border, const int bottom_border,
        cc->Inputs().Tag(kDetections).Get<std::vector<mediapipe::Detection>>();
    for (const auto& detection : raw_detections) {
      only_required_found = true;
-      MP_RETURN_IF_ERROR(UpdateRanges(detection, &xmin, &xmax, &ymin, &ymax));
+      MP_RETURN_IF_ERROR(UpdateRanges(
+          detection, options_.detection_shift_vertical(),
+          options_.detection_shift_horizontal(), &xmin, &xmax, &ymin, &ymax));
    }
  }

@@ -19,6 +19,7 @@ package mediapipe.autoflip;
 import "mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver.proto";
 import "mediapipe/framework/calculator.proto";

+// NextTag: 13
 message ContentZoomingCalculatorOptions {
  extend mediapipe.CalculatorOptions {
    optional ContentZoomingCalculatorOptions ext = 313091992;
@@ -44,6 +45,12 @@ message ContentZoomingCalculatorOptions {
    optional int64 height = 2;
  }
  optional Size target_size = 8;
+  // Amount to shift an input detection as a ratio of the size (positive:
+  // down/right, negative: up/left).  Use a negative value to increase padding
+  // above/left of an object, positive to increase padding below/right of an
+  // object.
+  optional float detection_shift_vertical = 11 [default = 0.0];
+  optional float detection_shift_horizontal = 12 [default = 0.0];

  // Deprecated parameters
  optional KinematicOptions kinematic_options = 2 [deprecated = true];
@@ -366,6 +366,45 @@ TEST(ContentZoomingCalculatorTest, ZoomTestNearInsideBorder) {
  CheckCropRect(42, 42, 83, 83, 1, runner->Outputs().Tag("CROP_RECT").packets);
 }

+TEST(ContentZoomingCalculatorTest, VerticalShift) {
+  auto config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfigD);
+  auto* options = config.mutable_options()->MutableExtension(
+      ContentZoomingCalculatorOptions::ext);
+  options->set_detection_shift_vertical(0.2);
+  auto runner = ::absl::make_unique<CalculatorRunner>(config);
+  AddDetection(cv::Rect_<float>(.1, .1, .1, .1), 0, runner.get());
+  MP_ASSERT_OK(runner->Run());
+  // 1000px * .1 offset + 1000*.1*.1 shift = 170
+  CheckCropRect(150, 170, 111, 111, 0,
+                runner->Outputs().Tag("CROP_RECT").packets);
+}
+
+TEST(ContentZoomingCalculatorTest, HorizontalShift) {
+  auto config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfigD);
+  auto* options = config.mutable_options()->MutableExtension(
+      ContentZoomingCalculatorOptions::ext);
+  options->set_detection_shift_horizontal(0.2);
+  auto runner = ::absl::make_unique<CalculatorRunner>(config);
+  AddDetection(cv::Rect_<float>(.1, .1, .1, .1), 0, runner.get());
+  MP_ASSERT_OK(runner->Run());
+  // 1000px * .1 offset + 1000*.1*.1 shift = 170
+  CheckCropRect(170, 150, 111, 111, 0,
+                runner->Outputs().Tag("CROP_RECT").packets);
+}
+
+TEST(ContentZoomingCalculatorTest, ShiftOutsideBounds) {
+  auto config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfigD);
+  auto* options = config.mutable_options()->MutableExtension(
+      ContentZoomingCalculatorOptions::ext);
+  options->set_detection_shift_vertical(-0.2);
+  options->set_detection_shift_horizontal(0.2);
+  auto runner = ::absl::make_unique<CalculatorRunner>(config);
+  AddDetection(cv::Rect_<float>(.9, 0, .1, .1), 0, runner.get());
+  MP_ASSERT_OK(runner->Run());
+  CheckCropRect(944, 56, 111, 111, 0,
+                runner->Outputs().Tag("CROP_RECT").packets);
+}
+
 }  // namespace
 }  // namespace autoflip

@@ -0,0 +1,60 @@
+# Copyright 2019 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+licenses(["notice"])  # Apache 2.0
+
+package(default_visibility = ["//mediapipe/examples:__subpackages__"])
+
+cc_binary(
+    name = "iris_depth_from_image_desktop",
+    srcs = ["iris_depth_from_image_desktop.cc"],
+    deps = [
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:image_frame",
+        "//mediapipe/framework/formats:image_frame_opencv",
+        "//mediapipe/framework/port:commandlineflags",
+        "//mediapipe/framework/port:file_helpers",
+        "//mediapipe/framework/port:opencv_highgui",
+        "//mediapipe/framework/port:opencv_imgproc",
+        "//mediapipe/framework/port:opencv_video",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/graphs/iris_tracking:iris_depth_cpu_deps",
+    ],
+)
+
+cc_binary(
+    name = "iris_tracking_cpu_video_input",
+    deps = [
+        "//mediapipe/examples/desktop:simple_run_graph_main",
+        "//mediapipe/graphs/iris_tracking:iris_tracking_cpu_video_input_deps",
+    ],
+)
+
+cc_binary(
+    name = "iris_tracking_cpu",
+    deps = [
+        "//mediapipe/examples/desktop:demo_run_graph_main",
+        "//mediapipe/graphs/iris_tracking:iris_tracking_cpu_deps",
+    ],
+)
+
+# Linux only
+cc_binary(
+    name = "iris_tracking_gpu",
+    deps = [
+        "//mediapipe/examples/desktop:demo_run_graph_main_gpu",
+        "//mediapipe/graphs/iris_tracking:iris_tracking_gpu_deps",
+    ],
+)
@@ -0,0 +1,162 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// A utility to extract iris depth from a single image of face using the graph
+// mediapipe/graphs/iris_tracking/iris_depth_cpu.pbtxt.
+#include <cstdlib>
+#include <memory>
+
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/image_frame.h"
+#include "mediapipe/framework/formats/image_frame_opencv.h"
+#include "mediapipe/framework/port/canonical_errors.h"
+#include "mediapipe/framework/port/commandlineflags.h"
+#include "mediapipe/framework/port/file_helpers.h"
+#include "mediapipe/framework/port/opencv_highgui_inc.h"
+#include "mediapipe/framework/port/opencv_imgproc_inc.h"
+#include "mediapipe/framework/port/opencv_video_inc.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/status.h"
+
+constexpr char kInputStream[] = "input_image_bytes";
+constexpr char kOutputImageStream[] = "output_image";
+constexpr char kLeftIrisDepthMmStream[] = "left_iris_depth_mm";
+constexpr char kRightIrisDepthMmStream[] = "right_iris_depth_mm";
+constexpr char kWindowName[] = "MediaPipe";
+constexpr char kCalculatorGraphConfigFile[] =
+    "mediapipe/graphs/iris_tracking/iris_depth_cpu.pbtxt";
+constexpr float kMicrosPerSecond = 1e6;
+
+DEFINE_string(input_image_path, "",
+              "Full path of image to load. "
+              "If not provided, nothing will run.");
+DEFINE_string(output_image_path, "",
+              "Full path of where to save image result (.jpg only). "
+              "If not provided, show result in a window.");
+
+namespace {
+
+::mediapipe::StatusOr<std::string> ReadFileToString(
+    const std::string& file_path) {
+  std::string contents;
+  MP_RETURN_IF_ERROR(::mediapipe::file::GetContents(file_path, &contents));
+  return contents;
+}
+
+::mediapipe::Status ProcessImage(
+    std::unique_ptr<::mediapipe::CalculatorGraph> graph) {
+  LOG(INFO) << "Load the image.";
+  ASSIGN_OR_RETURN(const std::string raw_image,
+                   ReadFileToString(FLAGS_input_image_path));
+
+  LOG(INFO) << "Start running the calculator graph.";
+  ASSIGN_OR_RETURN(::mediapipe::OutputStreamPoller output_image_poller,
+                   graph->AddOutputStreamPoller(kOutputImageStream));
+  ASSIGN_OR_RETURN(::mediapipe::OutputStreamPoller left_iris_depth_poller,
+                   graph->AddOutputStreamPoller(kLeftIrisDepthMmStream));
+  ASSIGN_OR_RETURN(::mediapipe::OutputStreamPoller right_iris_depth_poller,
+                   graph->AddOutputStreamPoller(kRightIrisDepthMmStream));
+  MP_RETURN_IF_ERROR(graph->StartRun({}));
+
+  // Send image packet into the graph.
+  const size_t fake_timestamp_us = (double)cv::getTickCount() /
+                                   (double)cv::getTickFrequency() *
+                                   kMicrosPerSecond;
+  MP_RETURN_IF_ERROR(graph->AddPacketToInputStream(
+      kInputStream, ::mediapipe::MakePacket<std::string>(raw_image).At(
+                        ::mediapipe::Timestamp(fake_timestamp_us))));
+
+  // Get the graph result packets, or stop if that fails.
+  ::mediapipe::Packet left_iris_depth_packet;
+  if (!left_iris_depth_poller.Next(&left_iris_depth_packet)) {
+    return ::mediapipe::UnknownError(
+        "Failed to get packet from output stream 'left_iris_depth_mm'.");
+  }
+  const auto& left_iris_depth_mm = left_iris_depth_packet.Get<float>();
+  const int left_iris_depth_cm = std::round(left_iris_depth_mm / 10);
+  std::cout << "Left Iris Depth: " << left_iris_depth_cm << " cm." << std::endl;
+
+  ::mediapipe::Packet right_iris_depth_packet;
+  if (!right_iris_depth_poller.Next(&right_iris_depth_packet)) {
+    return ::mediapipe::UnknownError(
+        "Failed to get packet from output stream 'right_iris_depth_mm'.");
+  }
+  const auto& right_iris_depth_mm = right_iris_depth_packet.Get<float>();
+  const int right_iris_depth_cm = std::round(right_iris_depth_mm / 10);
+  std::cout << "Right Iris Depth: " << right_iris_depth_cm << " cm."
+            << std::endl;
+
+  ::mediapipe::Packet output_image_packet;
+  if (!output_image_poller.Next(&output_image_packet)) {
+    return ::mediapipe::UnknownError(
+        "Failed to get packet from output stream 'output_image'.");
+  }
+  auto& output_frame = output_image_packet.Get<::mediapipe::ImageFrame>();
+
+  // Convert back to opencv for display or saving.
+  cv::Mat output_frame_mat = ::mediapipe::formats::MatView(&output_frame);
+  cv::cvtColor(output_frame_mat, output_frame_mat, cv::COLOR_RGB2BGR);
+  const bool save_image = !FLAGS_output_image_path.empty();
+  if (save_image) {
+    LOG(INFO) << "Saving image to file...";
+    cv::imwrite(FLAGS_output_image_path, output_frame_mat);
+  } else {
+    cv::namedWindow(kWindowName, /*flags=WINDOW_AUTOSIZE*/ 1);
+    cv::imshow(kWindowName, output_frame_mat);
+    // Press any key to exit.
+    cv::waitKey(0);
+  }
+
+  LOG(INFO) << "Shutting down.";
+  MP_RETURN_IF_ERROR(graph->CloseInputStream(kInputStream));
+  return graph->WaitUntilDone();
+}
+
+::mediapipe::Status RunMPPGraph() {
+  std::string calculator_graph_config_contents;
+  MP_RETURN_IF_ERROR(::mediapipe::file::GetContents(
+      kCalculatorGraphConfigFile, &calculator_graph_config_contents));
+  LOG(INFO) << "Get calculator graph config contents: "
+            << calculator_graph_config_contents;
+  ::mediapipe::CalculatorGraphConfig config =
+      ::mediapipe::ParseTextProtoOrDie<::mediapipe::CalculatorGraphConfig>(
+          calculator_graph_config_contents);
+
+  LOG(INFO) << "Initialize the calculator graph.";
+  std::unique_ptr<::mediapipe::CalculatorGraph> graph =
+      absl::make_unique<::mediapipe::CalculatorGraph>();
+  MP_RETURN_IF_ERROR(graph->Initialize(config));
+
+  const bool load_image = !FLAGS_input_image_path.empty();
+  if (load_image) {
+    return ProcessImage(std::move(graph));
+  } else {
+    return ::mediapipe::InvalidArgumentError("Missing image file.");
+  }
+}
+
+}  // namespace
+
+int main(int argc, char** argv) {
+  google::InitGoogleLogging(argv[0]);
+  gflags::ParseCommandLineFlags(&argc, &argv, true);
+  ::mediapipe::Status run_status = RunMPPGraph();
+  if (!run_status.ok()) {
+    LOG(ERROR) << "Failed to run the graph: " << run_status.message();
+    return EXIT_FAILURE;
+  } else {
+    LOG(INFO) << "Success!";
+  }
+  return EXIT_SUCCESS;
+}