Project import generated by Copybara.

GitOrigin-RevId: 72ff4ae24943c2ccf9905bc9e516042b0aa3dd86
2020-04-13 20:15:59 -04:00
parent 4c68eb4a70
commit 16e5d7242d
112 changed files with 4762 additions and 217 deletions
@@ -150,4 +150,34 @@ message ConversionOptions {
  optional int32 target_height = 2;
 }

-// TODO: Move other autoflip messages into this area.
+// Self-contained message that provides all needed information to render
+// autoflip with an external renderer.  One of these messages is required for
+// each frame of the video.
+message ExternalRenderFrame {
+  // Rectangle using opencv standard.
+  message Rect {
+    optional float x = 1;
+    optional float y = 2;
+    optional float width = 3;
+    optional float height = 4;
+  }
+  // RGB color [0...255]
+  message Color {
+    optional int32 r = 1;
+    optional int32 g = 2;
+    optional int32 b = 3;
+  }
+  // Rect that must be cropped out of the input frame.  It is in the
+  // original dimensions of the input video.  The first step to render this
+  // frame is to crop this rect from the input frame.
+  optional Rect crop_from_location = 1;
+  // The placement location where the above rect is placed on the output frame.
+  // This will always have the same aspect ratio as the above rect but scaling
+  // may be required.
+  optional Rect render_to_location = 2;
+  // If render_to_location is smaller than the output dimensions of the frame,
+  // fill the rest of the frame with this color.
+  optional Color padding_color = 3;
+  // Timestamp in microseconds of this frame.
+  optional uint64 timestamp_us = 4;
+}
@@ -44,11 +44,19 @@ constexpr char kInputExternalSettings[] = "EXTERNAL_SETTINGS";
 // TargetSizeType::MAXIMIZE_TARGET_DIMENSION
 constexpr char kAspectRatio[] = "EXTERNAL_ASPECT_RATIO";

+// Output the cropped frames, as well as visualization of crop regions and focus
+// points. Note that, KEY_FRAME_CROP_REGION_VIZ_FRAMES and
+// SALIENT_POINT_FRAME_VIZ_FRAMES can only be enabled when CROPPED_FRAMES is
+// enabled.
 constexpr char kOutputCroppedFrames[] = "CROPPED_FRAMES";
 constexpr char kOutputKeyFrameCropViz[] = "KEY_FRAME_CROP_REGION_VIZ_FRAMES";
 constexpr char kOutputFocusPointFrameViz[] = "SALIENT_POINT_FRAME_VIZ_FRAMES";
 constexpr char kOutputSummary[] = "CROPPING_SUMMARY";

+// External rendering outputs
+constexpr char kExternalRenderingPerFrame[] = "EXTERNAL_RENDERING_PER_FRAME";
+constexpr char kExternalRenderingFullVid[] = "EXTERNAL_RENDERING_FULL_VID";
+
 ::mediapipe::Status SceneCroppingCalculator::GetContract(
    ::mediapipe::CalculatorContract* cc) {
  if (cc->InputSidePackets().HasTag(kInputExternalSettings)) {
@@ -67,16 +75,36 @@ constexpr char kOutputSummary[] = "CROPPING_SUMMARY";
  }
  cc->Inputs().Tag(kInputShotBoundaries).Set<bool>();

-  cc->Outputs().Tag(kOutputCroppedFrames).Set<ImageFrame>();
+  if (cc->Outputs().HasTag(kOutputCroppedFrames)) {
+    cc->Outputs().Tag(kOutputCroppedFrames).Set<ImageFrame>();
+  }
  if (cc->Outputs().HasTag(kOutputKeyFrameCropViz)) {
+    RET_CHECK(cc->Outputs().HasTag(kOutputCroppedFrames))
+        << "KEY_FRAME_CROP_REGION_VIZ_FRAMES can only be used when "
+           "CROPPED_FRAMES is specified.";
    cc->Outputs().Tag(kOutputKeyFrameCropViz).Set<ImageFrame>();
  }
  if (cc->Outputs().HasTag(kOutputFocusPointFrameViz)) {
+    RET_CHECK(cc->Outputs().HasTag(kOutputCroppedFrames))
+        << "SALIENT_POINT_FRAME_VIZ_FRAMES can only be used when "
+           "CROPPED_FRAMES is specified.";
    cc->Outputs().Tag(kOutputFocusPointFrameViz).Set<ImageFrame>();
  }
  if (cc->Outputs().HasTag(kOutputSummary)) {
    cc->Outputs().Tag(kOutputSummary).Set<VideoCroppingSummary>();
  }
+  if (cc->Outputs().HasTag(kExternalRenderingPerFrame)) {
+    cc->Outputs().Tag(kExternalRenderingPerFrame).Set<ExternalRenderFrame>();
+  }
+  if (cc->Outputs().HasTag(kExternalRenderingFullVid)) {
+    cc->Outputs()
+        .Tag(kExternalRenderingFullVid)
+        .Set<std::vector<ExternalRenderFrame>>();
+  }
+  RET_CHECK(cc->Outputs().HasTag(kExternalRenderingPerFrame) ||
+            cc->Outputs().HasTag(kExternalRenderingFullVid) ||
+            cc->Outputs().HasTag(kOutputCroppedFrames))
+      << "At leaset one output stream must be specified";
  return ::mediapipe::OkStatus();
 }

@@ -104,6 +132,11 @@ constexpr char kOutputSummary[] = "CROPPING_SUMMARY";
  if (cc->Outputs().HasTag(kOutputSummary)) {
    summary_ = absl::make_unique<VideoCroppingSummary>();
  }
+  if (cc->Outputs().HasTag(kExternalRenderingFullVid)) {
+    external_render_list_ =
+        absl::make_unique<std::vector<ExternalRenderFrame>>();
+  }
+  should_perform_frame_cropping_ = cc->Outputs().HasTag(kOutputCroppedFrames);
  return ::mediapipe::OkStatus();
 }

@@ -127,6 +160,28 @@ namespace {
  *aspect_ratio = width_ratio / height_ratio;
  return ::mediapipe::OkStatus();
 }
+void ConstructExternalRenderMessage(
+    const cv::Rect& crop_from_location, const cv::Rect& render_to_location,
+    const cv::Scalar& padding_color, const uint64 timestamp_us,
+    ExternalRenderFrame* external_render_message) {
+  auto crop_from_message =
+      external_render_message->mutable_crop_from_location();
+  crop_from_message->set_x(crop_from_location.x);
+  crop_from_message->set_y(crop_from_location.y);
+  crop_from_message->set_width(crop_from_location.width);
+  crop_from_message->set_height(crop_from_location.height);
+  auto render_to_message =
+      external_render_message->mutable_render_to_location();
+  render_to_message->set_x(render_to_location.x);
+  render_to_message->set_y(render_to_location.y);
+  render_to_message->set_width(render_to_location.width);
+  render_to_message->set_height(render_to_location.height);
+  auto padding_color_message = external_render_message->mutable_padding_color();
+  padding_color_message->set_r(padding_color[0]);
+  padding_color_message->set_g(padding_color[1]);
+  padding_color_message->set_b(padding_color[2]);
+  external_render_message->set_timestamp_us(timestamp_us);
+}
 }  // namespace

 ::mediapipe::Status SceneCroppingCalculator::Process(
@@ -230,8 +285,9 @@ namespace {
    is_end_of_scene = cc->Inputs().Tag(kInputShotBoundaries).Get<bool>();
  }
  const bool force_buffer_flush =
-      scene_frames_.size() >= options_.max_scene_size();
-  if (!scene_frames_.empty() && (is_end_of_scene || force_buffer_flush)) {
+      scene_frame_timestamps_.size() >= options_.max_scene_size();
+  if (!scene_frame_timestamps_.empty() &&
+      (is_end_of_scene || force_buffer_flush)) {
    MP_RETURN_IF_ERROR(ProcessScene(is_end_of_scene, cc));
  }

@@ -240,11 +296,14 @@ namespace {
    LOG_EVERY_N(ERROR, 10)
        << "------------------------ (Breathing) Time(s): "
        << cc->Inputs().Tag(kInputVideoFrames).Value().Timestamp().Seconds();
-    const auto& frame = cc->Inputs().Tag(kInputVideoFrames).Get<ImageFrame>();
-    const cv::Mat frame_mat = formats::MatView(&frame);
-    cv::Mat copy_mat;
-    frame_mat.copyTo(copy_mat);
-    scene_frames_.push_back(copy_mat);
+    // Only buffer frames if |should_perform_frame_cropping_| is true.
+    if (should_perform_frame_cropping_) {
+      const auto& frame = cc->Inputs().Tag(kInputVideoFrames).Get<ImageFrame>();
+      const cv::Mat frame_mat = formats::MatView(&frame);
+      cv::Mat copy_mat;
+      frame_mat.copyTo(copy_mat);
+      scene_frames_or_empty_.push_back(copy_mat);
+    }
    scene_frame_timestamps_.push_back(cc->InputTimestamp().Value());
    is_key_frames_.push_back(
        !cc->Inputs().Tag(kInputDetections).Value().IsEmpty());
@@ -274,7 +333,7 @@ namespace {

 ::mediapipe::Status SceneCroppingCalculator::Close(
    ::mediapipe::CalculatorContext* cc) {
-  if (!scene_frames_.empty()) {
+  if (!scene_frame_timestamps_.empty()) {
    MP_RETURN_IF_ERROR(ProcessScene(/* is_end_of_scene = */ true, cc));
  }
  if (cc->Outputs().HasTag(kOutputSummary)) {
@@ -282,16 +341,25 @@ namespace {
        .Tag(kOutputSummary)
        .Add(summary_.release(), Timestamp::PostStream());
  }
+  if (cc->Outputs().HasTag(kExternalRenderingFullVid)) {
+    cc->Outputs()
+        .Tag(kExternalRenderingFullVid)
+        .Add(external_render_list_.release(), Timestamp::PostStream());
+  }
  return ::mediapipe::OkStatus();
 }

-::mediapipe::Status SceneCroppingCalculator::RemoveStaticBorders() {
-  int top_border_size = 0, bottom_border_size = 0;
+// TODO: split this function into two, one for calculating the border
+// sizes, the other for the actual removal of borders from the frames.
+::mediapipe::Status SceneCroppingCalculator::RemoveStaticBorders(
+    int* top_border_size, int* bottom_border_size) {
+  *top_border_size = 0;
+  *bottom_border_size = 0;
  MP_RETURN_IF_ERROR(ComputeSceneStaticBordersSize(
-      static_features_, &top_border_size, &bottom_border_size));
+      static_features_, top_border_size, bottom_border_size));
  const double scale = static_cast<double>(frame_height_) / key_frame_height_;
-  top_border_distance_ = std::round(scale * top_border_size);
-  const int bottom_border_distance = std::round(scale * bottom_border_size);
+  top_border_distance_ = std::round(scale * *top_border_size);
+  const int bottom_border_distance = std::round(scale * *bottom_border_size);
  effective_frame_height_ =
      frame_height_ - top_border_distance_ - bottom_border_distance;

@@ -301,10 +369,10 @@ namespace {
    // Remove borders from frames.
    cv::Rect roi(0, top_border_distance_, frame_width_,
                 effective_frame_height_);
-    for (int i = 0; i < scene_frames_.size(); ++i) {
+    for (int i = 0; i < scene_frames_or_empty_.size(); ++i) {
      cv::Mat tmp;
-      scene_frames_[i](roi).copyTo(tmp);
-      scene_frames_[i] = tmp;
+      scene_frames_or_empty_[i](roi).copyTo(tmp);
+      scene_frames_or_empty_[i] = tmp;
    }
    // Adjust detection bounding boxes.
    for (int i = 0; i < key_frame_infos_.size(); ++i) {
@@ -373,7 +441,9 @@ void SceneCroppingCalculator::FilterKeyFrameInfo() {
  FilterKeyFrameInfo();

  // Removes any static borders.
-  MP_RETURN_IF_ERROR(RemoveStaticBorders());
+  int top_static_border_size, bottom_static_border_size;
+  MP_RETURN_IF_ERROR(
+      RemoveStaticBorders(&top_static_border_size, &bottom_static_border_size));

  // Decides if solid background color padding is possible and sets up color
  // interpolation functions in CIELAB. Uses linear interpolation by default.
@@ -409,21 +479,32 @@ void SceneCroppingCalculator::FilterKeyFrameInfo() {

  // Crops scene frames.
  std::vector<cv::Mat> cropped_frames;
+  std::vector<cv::Rect> crop_from_locations;
+
+  auto* cropped_frames_ptr =
+      should_perform_frame_cropping_ ? &cropped_frames : nullptr;
+
  MP_RETURN_IF_ERROR(scene_cropper_->CropFrames(
-      scene_summary, scene_frames_, focus_point_frames,
-      prior_focus_point_frames_, &cropped_frames));
+      scene_summary, scene_frame_timestamps_.size(), scene_frames_or_empty_,
+      focus_point_frames, prior_focus_point_frames_, top_static_border_size,
+      bottom_static_border_size, &crop_from_locations, cropped_frames_ptr));

  // Formats and outputs cropped frames.
  bool apply_padding = false;
  float vertical_fill_precent;
-  MP_RETURN_IF_ERROR(FormatAndOutputCroppedFrames(
-      cropped_frames, &apply_padding, &vertical_fill_precent, cc));
-
+  std::vector<cv::Rect> render_to_locations;
+  cv::Scalar padding_color;
+  if (should_perform_frame_cropping_) {
+    MP_RETURN_IF_ERROR(FormatAndOutputCroppedFrames(
+        cropped_frames, &render_to_locations, &apply_padding, &padding_color,
+        &vertical_fill_precent, cc));
+  }
  // Caches prior FocusPointFrames if this was not the end of a scene.
  prior_focus_point_frames_.clear();
  if (!is_end_of_scene) {
-    const int start = std::max(0, static_cast<int>(scene_frames_.size()) -
-                                      options_.prior_frame_buffer_size());
+    const int start =
+        std::max(0, static_cast<int>(scene_frame_timestamps_.size()) -
+                        options_.prior_frame_buffer_size());
    for (int i = start; i < num_key_frames; ++i) {
      prior_focus_point_frames_.push_back(focus_point_frames[i]);
    }
@@ -449,8 +530,31 @@ void SceneCroppingCalculator::FilterKeyFrameInfo() {
    scene_summary->set_is_padded(apply_padding);
  }

+  if (cc->Outputs().HasTag(kExternalRenderingPerFrame)) {
+    for (int i = 0; i < scene_frame_timestamps_.size(); i++) {
+      auto external_render_message = absl::make_unique<ExternalRenderFrame>();
+      ConstructExternalRenderMessage(
+          crop_from_locations[i], render_to_locations[i], padding_color,
+          scene_frame_timestamps_[i], external_render_message.get());
+      cc->Outputs()
+          .Tag(kExternalRenderingPerFrame)
+          .Add(external_render_message.release(),
+               Timestamp(scene_frame_timestamps_[i]));
+    }
+  }
+
+  if (cc->Outputs().HasTag(kExternalRenderingFullVid)) {
+    for (int i = 0; i < scene_frame_timestamps_.size(); i++) {
+      ExternalRenderFrame render_frame;
+      ConstructExternalRenderMessage(crop_from_locations[i],
+                                     render_to_locations[i], padding_color,
+                                     scene_frame_timestamps_[i], &render_frame);
+      external_render_list_->push_back(render_frame);
+    }
+  }
+
  key_frame_infos_.clear();
-  scene_frames_.clear();
+  scene_frames_or_empty_.clear();
  scene_frame_timestamps_.clear();
  is_key_frames_.clear();
  static_features_.clear();
@@ -459,8 +563,10 @@ void SceneCroppingCalculator::FilterKeyFrameInfo() {
 }

 ::mediapipe::Status SceneCroppingCalculator::FormatAndOutputCroppedFrames(
-    const std::vector<cv::Mat>& cropped_frames, bool* apply_padding,
-    float* vertical_fill_precent, CalculatorContext* cc) {
+    const std::vector<cv::Mat>& cropped_frames,
+    std::vector<cv::Rect>* render_to_locations, bool* apply_padding,
+    cv::Scalar* padding_color, float* vertical_fill_precent,
+    CalculatorContext* cc) {
  RET_CHECK(apply_padding) << "Has padding boolean is null.";
  if (cropped_frames.empty()) {
    return ::mediapipe::OkStatus();
@@ -493,10 +599,22 @@ void SceneCroppingCalculator::FilterKeyFrameInfo() {
            << " target height = " << target_height_;
  }

+  // Compute the "render to" location.  This is where the rect taken from the
+  // input video gets pasted on the output frame.  For use with external
+  // rendering solutions.
+  const int num_frames = cropped_frames.size();
+  for (int i = 0; i < num_frames; i++) {
+    if (*apply_padding) {
+      render_to_locations->push_back(padder_->ComputeOutputLocation());
+    } else {
+      render_to_locations->push_back(
+          cv::Rect(0, 0, target_width_, target_height_));
+    }
+  }
+
  // Resizes cropped frames, pads frames, and output frames.
  cv::Scalar* background_color = nullptr;
  cv::Scalar interpolated_color;
-  const int num_frames = cropped_frames.size();
  for (int i = 0; i < num_frames; ++i) {
    const int64 time_ms = scene_frame_timestamps_[i];
    const Timestamp timestamp(time_ms);
@@ -561,9 +679,9 @@ mediapipe::Status SceneCroppingCalculator::OutputVizFrames(
  if (cc->Outputs().HasTag(kOutputKeyFrameCropViz)) {
    std::vector<std::unique_ptr<ImageFrame>> viz_frames;
    MP_RETURN_IF_ERROR(DrawDetectionsAndCropRegions(
-        scene_frames_, is_key_frames_, key_frame_infos_, key_frame_crop_results,
-        frame_format_, &viz_frames));
-    for (int i = 0; i < scene_frames_.size(); ++i) {
+        scene_frames_or_empty_, is_key_frames_, key_frame_infos_,
+        key_frame_crop_results, frame_format_, &viz_frames));
+    for (int i = 0; i < scene_frames_or_empty_.size(); ++i) {
      cc->Outputs()
          .Tag(kOutputKeyFrameCropViz)
          .Add(viz_frames[i].release(), Timestamp(scene_frame_timestamps_[i]));
@@ -572,9 +690,10 @@ mediapipe::Status SceneCroppingCalculator::OutputVizFrames(
  if (cc->Outputs().HasTag(kOutputFocusPointFrameViz)) {
    std::vector<std::unique_ptr<ImageFrame>> viz_frames;
    MP_RETURN_IF_ERROR(DrawFocusPointAndCropWindow(
-        scene_frames_, focus_point_frames, options_.viz_overlay_opacity(),
-        crop_window_width, crop_window_height, frame_format_, &viz_frames));
-    for (int i = 0; i < scene_frames_.size(); ++i) {
+        scene_frames_or_empty_, focus_point_frames,
+        options_.viz_overlay_opacity(), crop_window_width, crop_window_height,
+        frame_format_, &viz_frames));
+    for (int i = 0; i < scene_frames_or_empty_.size(); ++i) {
      cc->Outputs()
          .Tag(kOutputFocusPointFrameViz)
          .Add(viz_frames[i].release(), Timestamp(scene_frame_timestamps_[i]));
@@ -79,8 +79,10 @@ namespace autoflip {
 //     Indicators for shot boundaries (output of shot boundary detection).
 // - optional tag KEY_FRAMES (type ImageFrame):
 //     Key frames on which features are detected. This is only used to set the
-//     detection features frame size, and when it is omitted, the features frame
-//     size is assumed to be the original scene frame size.
+//     detection features frame size.  Alternatively, set
+//     video_feature_width/video_features_height within the options proto to
+//     define this value.  When neither is set, the features frame size is
+//     assumed to be the original scene frame size.
 //
 // Output streams:
 // - required tag CROPPED_FRAMES (type ImageFrame):
@@ -95,6 +97,12 @@ namespace autoflip {
 // - optional tag CROPPING_SUMMARY (type VideoCroppingSummary):
 //     Debug summary information for the video. Only generates one packet when
 //     calculator closes.
+// - optional tag EXTERNAL_RENDERING_PER_FRAME (type ExternalRenderFrame)
+//     Provides a per-frame message that can be used to render autoflip using an
+//     external renderer.
+// - optional tag EXTERNAL_RENDERING_FULL_VID (type Vector<ExternalRenderFrame>)
+//     Provides an end-stream message that can be used to render autoflip using
+//     an external renderer.
 //
 // Example config:
 // node {
@@ -134,8 +142,11 @@ class SceneCroppingCalculator : public CalculatorBase {
  ::mediapipe::Status Close(::mediapipe::CalculatorContext* cc) override;

 private:
-  // Removes any static borders from the scene frames before cropping.
-  ::mediapipe::Status RemoveStaticBorders();
+  // Removes any static borders from the scene frames before cropping. The
+  // arguments |top_border_size| and |bottom_border_size| report the size of the
+  // removed borders.
+  ::mediapipe::Status RemoveStaticBorders(int* top_border_size,
+                                          int* bottom_border_size);

  // Initializes a FrameCropRegionComputer given input and target frame sizes.
  ::mediapipe::Status InitializeFrameCropRegionComputer();
@@ -158,8 +169,10 @@ class SceneCroppingCalculator : public CalculatorBase {
  // solid background from static features if possible, otherwise uses blurred
  // background. Sets apply_padding to true if the scene is padded.
  ::mediapipe::Status FormatAndOutputCroppedFrames(
-      const std::vector<cv::Mat>& cropped_frames, bool* apply_padding,
-      float* vertical_fill_precent, CalculatorContext* cc);
+      const std::vector<cv::Mat>& cropped_frames,
+      std::vector<cv::Rect>* render_to_locations, bool* apply_padding,
+      cv::Scalar* padding_color, float* vertical_fill_precent,
+      CalculatorContext* cc);

  // Draws and outputs visualization frames if those streams are present.
  ::mediapipe::Status OutputVizFrames(
@@ -193,7 +206,11 @@ class SceneCroppingCalculator : public CalculatorBase {

  // Buffered frames, timestamps, and indicators for key frames in the current
  // scene (size = number of input video frames).
-  std::vector<cv::Mat> scene_frames_;
+  // Note: scene_frames_or_empty_ may be empty if the actual cropping operation
+  // of frames is turned off, e.g. when |should_perform_frame_cropping_| is
+  // false, so rely on scene_frame_timestamps_.size() to query the number of
+  // accumulated timestamps rather than scene_frames_or_empty_.size().
+  std::vector<cv::Mat> scene_frames_or_empty_;
  std::vector<int64> scene_frame_timestamps_;
  std::vector<bool> is_key_frames_;

@@ -242,6 +259,17 @@ class SceneCroppingCalculator : public CalculatorBase {

  // Optional diagnostic summary output emitted in Close().
  std::unique_ptr<VideoCroppingSummary> summary_ = nullptr;
+
+  // Optional list of external rendering messages for each processed frame.
+  std::unique_ptr<std::vector<ExternalRenderFrame>> external_render_list_;
+
+  // Determines whether to perform real cropping on input frames. This flag is
+  // useful when the user only needs to compute cropping windows, in which case
+  // setting this flag to false can avoid buffering as well as cropping frames.
+  // This can significantly reduce memory usage and speed up processing. Some
+  // debugging visualization inevitably will be disabled because of this flag
+  // too.
+  bool should_perform_frame_cropping_ = false;
 };
 }  // namespace autoflip
 }  // namespace mediapipe
@@ -68,6 +68,22 @@ constexpr char kNoKeyFrameConfig[] = R"(
    }
  })";

+constexpr char kDebugConfigNoCroppedFrame[] = R"(
+  calculator: "SceneCroppingCalculator"
+  input_stream: "VIDEO_FRAMES:camera_frames_org"
+  input_stream: "KEY_FRAMES:down_sampled_frames"
+  input_stream: "DETECTION_FEATURES:salient_regions"
+  input_stream: "STATIC_FEATURES:border_features"
+  input_stream: "SHOT_BOUNDARIES:shot_boundary_frames"
+  output_stream: "KEY_FRAME_CROP_REGION_VIZ_FRAMES:key_frame_crop_viz_frames"
+  output_stream: "SALIENT_POINT_FRAME_VIZ_FRAMES:salient_point_viz_frames"
+  options: {
+    [mediapipe.autoflip.SceneCroppingCalculatorOptions.ext]: {
+      target_width: $0
+      target_height: $1
+    }
+  })";
+
 constexpr char kDebugConfig[] = R"(
  calculator: "SceneCroppingCalculator"
  input_stream: "VIDEO_FRAMES:camera_frames_org"
@@ -79,6 +95,8 @@ constexpr char kDebugConfig[] = R"(
  output_stream: "KEY_FRAME_CROP_REGION_VIZ_FRAMES:key_frame_crop_viz_frames"
  output_stream: "SALIENT_POINT_FRAME_VIZ_FRAMES:salient_point_viz_frames"
  output_stream: "CROPPING_SUMMARY:cropping_summaries"
+  output_stream: "EXTERNAL_RENDERING_PER_FRAME:external_rendering_per_frame"
+  output_stream: "EXTERNAL_RENDERING_FULL_VID:external_rendering_full_vid"
  options: {
    [mediapipe.autoflip.SceneCroppingCalculatorOptions.ext]: {
      target_width: $0
@@ -257,6 +275,17 @@ TEST(SceneCroppingCalculatorTest, ChecksPriorFrameBufferSize) {
              HasSubstr("Prior frame buffer size is negative."));
 }

+TEST(SceneCroppingCalculatorTest, ChecksDebugConfigWithoutCroppedFrame) {
+  const CalculatorGraphConfig::Node config =
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(absl::Substitute(
+          kDebugConfigNoCroppedFrame, kTargetWidth, kTargetHeight,
+          kTargetSizeType, 0, kPriorFrameBufferSize));
+  auto runner = absl::make_unique<CalculatorRunner>(config);
+  const auto status = runner->Run();
+  EXPECT_FALSE(status.ok());
+  EXPECT_THAT(status.ToString(), HasSubstr("can only be used when"));
+}
+
 // Checks that the calculator crops scene frames when there is no input key
 // frames stream.
 TEST(SceneCroppingCalculatorTest, HandlesNoKeyFrames) {
@@ -299,14 +328,34 @@ TEST(SceneCroppingCalculatorTest, OutputsDebugStreams) {
  EXPECT_TRUE(outputs.HasTag("KEY_FRAME_CROP_REGION_VIZ_FRAMES"));
  EXPECT_TRUE(outputs.HasTag("SALIENT_POINT_FRAME_VIZ_FRAMES"));
  EXPECT_TRUE(outputs.HasTag("CROPPING_SUMMARY"));
+  EXPECT_TRUE(outputs.HasTag("EXTERNAL_RENDERING_PER_FRAME"));
+  EXPECT_TRUE(outputs.HasTag("EXTERNAL_RENDERING_FULL_VID"));
  const auto& crop_region_viz_frames_outputs =
      outputs.Tag("KEY_FRAME_CROP_REGION_VIZ_FRAMES").packets;
  const auto& salient_point_viz_frames_outputs =
      outputs.Tag("SALIENT_POINT_FRAME_VIZ_FRAMES").packets;
  const auto& summary_output = outputs.Tag("CROPPING_SUMMARY").packets;
+  const auto& ext_render_per_frame =
+      outputs.Tag("EXTERNAL_RENDERING_PER_FRAME").packets;
+  const auto& ext_render_full_vid =
+      outputs.Tag("EXTERNAL_RENDERING_FULL_VID").packets;
  EXPECT_EQ(crop_region_viz_frames_outputs.size(), num_frames);
  EXPECT_EQ(salient_point_viz_frames_outputs.size(), num_frames);
  EXPECT_EQ(summary_output.size(), 1);
+  EXPECT_EQ(ext_render_per_frame.size(), num_frames);
+  EXPECT_EQ(ext_render_full_vid.size(), 1);
+  EXPECT_EQ(ext_render_per_frame[0].Get<ExternalRenderFrame>().timestamp_us(),
+            0);
+  EXPECT_EQ(ext_render_full_vid[0]
+                .Get<std::vector<ExternalRenderFrame>>()[0]
+                .timestamp_us(),
+            0);
+  EXPECT_EQ(ext_render_per_frame[1].Get<ExternalRenderFrame>().timestamp_us(),
+            20000);
+  EXPECT_EQ(ext_render_full_vid[0]
+                .Get<std::vector<ExternalRenderFrame>>()[1]
+                .timestamp_us(),
+            20000);

  for (int i = 0; i < num_frames; ++i) {
    const auto& crop_region_viz_frame =
@@ -173,5 +173,28 @@ PaddingEffectGenerator::PaddingEffectGenerator(const int input_width,
  return ::mediapipe::OkStatus();
 }

+cv::Rect PaddingEffectGenerator::ComputeOutputLocation() {
+  const int effective_input_width =
+      is_vertical_padding_ ? input_width_ : input_height_;
+  const int effective_input_height =
+      is_vertical_padding_ ? input_height_ : input_width_;
+  const int effective_output_width =
+      is_vertical_padding_ ? output_width_ : output_height_;
+  const int effective_output_height =
+      is_vertical_padding_ ? output_height_ : output_width_;
+
+  // Step 3 from "process" call above, compute foreground location.
+  const int foreground_height =
+      effective_input_height * effective_output_width / effective_input_width;
+  const int x = 0;
+  const int y = (effective_output_height - foreground_height) / 2;
+  const int width = effective_output_width;
+  const int height = foreground_height;
+
+  cv::Rect region_to_embed_foreground(x, y, width, height);
+
+  return region_to_embed_foreground;
+}
+
 }  // namespace autoflip
 }  // namespace mediapipe
@@ -55,6 +55,10 @@ class PaddingEffectGenerator {
      ImageFrame* output_frame,
      const cv::Scalar* background_color_in_rgb = nullptr);

+  // Compute the "render location" on the output frame where the "crop from"
+  // location is to be placed.  For use with external rendering soutions.
+  cv::Rect ComputeOutputLocation();
+
 private:
  double target_aspect_ratio_;
  int input_width_ = -1;
@@ -182,6 +182,16 @@ TEST(PaddingEffectGeneratorTest, ScaleToMultipleOfTwo) {
  EXPECT_EQ(result_frame.Width(), expect_width);
  EXPECT_EQ(result_frame.Height(), expect_height);
 }
+
+TEST(PaddingEffectGeneratorTest, ComputeOutputLocation) {
+  PaddingEffectGenerator generator(1920, 1080, 1.0);
+
+  auto result_rect = generator.ComputeOutputLocation();
+  EXPECT_EQ(result_rect.x, 0);
+  EXPECT_EQ(result_rect.y, 236);
+  EXPECT_EQ(result_rect.width, 1080);
+  EXPECT_EQ(result_rect.height, 607);
+}
 }  // namespace
 }  // namespace autoflip
 }  // namespace mediapipe
@@ -25,14 +25,13 @@ namespace mediapipe {
 namespace autoflip {

 ::mediapipe::Status SceneCropper::CropFrames(
-    const SceneKeyFrameCropSummary& scene_summary,
-    const std::vector<cv::Mat>& scene_frames,
+    const SceneKeyFrameCropSummary& scene_summary, const int num_scene_frames,
+    const std::vector<cv::Mat>& scene_frames_or_empty,
    const std::vector<FocusPointFrame>& focus_point_frames,
    const std::vector<FocusPointFrame>& prior_focus_point_frames,
+    int top_static_border_size, int bottom_static_border_size,
+    std::vector<cv::Rect>* crop_from_location,
    std::vector<cv::Mat>* cropped_frames) const {
-  RET_CHECK_NE(cropped_frames, nullptr) << "Output cropped frames is null.";
-
-  const int num_scene_frames = scene_frames.size();
  RET_CHECK_GT(num_scene_frames, 0) << "No scene frames.";
  RET_CHECK_EQ(focus_point_frames.size(), num_scene_frames)
      << "Wrong size of FocusPointFrames.";
@@ -69,15 +68,36 @@ namespace autoflip {
    xform = affine_opencv;
  }

+  // If no cropped_frames is passed in, return directly.
+  if (!cropped_frames) {
+    return ::mediapipe::OkStatus();
+  }
+  RET_CHECK(!scene_frames_or_empty.empty())
+      << "If |cropped_frames| != nullptr, scene_frames_or_empty must not be "
+         "empty.";
  // Prepares cropped frames.
  cropped_frames->resize(num_scene_frames);
  for (int i = 0; i < num_scene_frames; ++i) {
-    (*cropped_frames)[i] =
-        cv::Mat::zeros(crop_height, crop_width, scene_frames[i].type());
+    (*cropped_frames)[i] = cv::Mat::zeros(crop_height, crop_width,
+                                          scene_frames_or_empty[i].type());
  }

-  return AffineRetarget(cv::Size(crop_width, crop_height), scene_frames,
-                        scene_frame_xforms, cropped_frames);
+  // Store the "crop from" location on the input frame for use with an external
+  // renderer.
+  for (int i = 0; i < num_scene_frames; i++) {
+    const int left = scene_frame_xforms[i].at<float>(0, 2);
+    const int right = left + crop_width;
+    const int top = top_static_border_size;
+    const int bottom =
+        top_static_border_size +
+        (crop_height - top_static_border_size - bottom_static_border_size);
+    crop_from_location->push_back(
+        cv::Rect(left, top, right - left, bottom - top));
+  }
+
+  return AffineRetarget(cv::Size(crop_width, crop_height),
+                        scene_frames_or_empty, scene_frame_xforms,
+                        cropped_frames);
 }

 }  // namespace autoflip
@@ -48,14 +48,19 @@ class SceneCropper {
  SceneCropper() {}
  ~SceneCropper() {}

-  // Crops scene frames given SceneKeyFrameCropSummary, FocusPointFrames, and
-  // any prior FocusPointFrames (to ensure smoothness when there was no actual
-  // scene change).
+  // Computes transformation matrix given SceneKeyFrameCropSummary,
+  // FocusPointFrames, and any prior FocusPointFrames (to ensure smoothness when
+  // there was no actual scene change). Optionally crops the input frames based
+  // on the transform matrix if |cropped_frames| is not nullptr and
+  // |scene_frames_or_empty| isn't empty.
+  // TODO: split this function into two separate functions.
  ::mediapipe::Status CropFrames(
-      const SceneKeyFrameCropSummary& scene_summary,
-      const std::vector<cv::Mat>& scene_frames,
+      const SceneKeyFrameCropSummary& scene_summary, const int num_scene_frames,
+      const std::vector<cv::Mat>& scene_frames_or_empty,
      const std::vector<FocusPointFrame>& focus_point_frames,
      const std::vector<FocusPointFrame>& prior_focus_point_frames,
+      int top_static_border_size, int bottom_static_border_size,
+      std::vector<cv::Rect>* all_scene_frame_xforms,
      std::vector<cv::Mat>* cropped_frames) const;
 };

@@ -71,24 +71,16 @@ std::vector<FocusPointFrame> GetDefaultFocusPointFrames() {
  return GetFocusPointFrames(kNumSceneFrames);
 }

-// Checks that CropFrames checks output pointer is not null.
-TEST(SceneCropperTest, CropFramesChecksOutputNotNull) {
-  SceneCropper scene_cropper;
-  const auto status = scene_cropper.CropFrames(
-      GetDefaultSceneKeyFrameCropSummary(), GetDefaultSceneFrames(),
-      GetDefaultFocusPointFrames(), GetFocusPointFrames(0), nullptr);
-  EXPECT_FALSE(status.ok());
-  EXPECT_THAT(status.ToString(), HasSubstr("Output cropped frames is null."));
-}
-
 // Checks that CropFrames checks that scene frames size is positive.
 TEST(SceneCropperTest, CropFramesChecksSceneFramesSize) {
  SceneCropper scene_cropper;
  std::vector<cv::Mat> scene_frames(0);
  std::vector<cv::Mat> cropped_frames;
+  std::vector<cv::Rect> crop_from_locations;
  const auto status = scene_cropper.CropFrames(
-      GetDefaultSceneKeyFrameCropSummary(), scene_frames,
-      GetDefaultFocusPointFrames(), GetFocusPointFrames(0), &cropped_frames);
+      GetDefaultSceneKeyFrameCropSummary(), scene_frames.size(), scene_frames,
+      GetDefaultFocusPointFrames(), GetFocusPointFrames(0), 0, 0,
+      &crop_from_locations, &cropped_frames);
  EXPECT_FALSE(status.ok());
  EXPECT_THAT(status.ToString(), HasSubstr("No scene frames."));
 }
@@ -97,10 +89,12 @@ TEST(SceneCropperTest, CropFramesChecksSceneFramesSize) {
 TEST(SceneCropperTest, CropFramesChecksFocusPointFramesSize) {
  SceneCropper scene_cropper;
  std::vector<cv::Mat> cropped_frames;
+  std::vector<cv::Rect> crop_from_locations;
+  const auto& scene_frames = GetDefaultSceneFrames();
  const auto status = scene_cropper.CropFrames(
-      GetDefaultSceneKeyFrameCropSummary(), GetDefaultSceneFrames(),
-      GetFocusPointFrames(kNumSceneFrames - 1), GetFocusPointFrames(0),
-      &cropped_frames);
+      GetDefaultSceneKeyFrameCropSummary(), scene_frames.size(), scene_frames,
+      GetFocusPointFrames(kNumSceneFrames - 1), GetFocusPointFrames(0), 0, 0,
+      &crop_from_locations, &cropped_frames);
  EXPECT_FALSE(status.ok());
  EXPECT_THAT(status.ToString(), HasSubstr("Wrong size of FocusPointFrames"));
 }
@@ -111,9 +105,12 @@ TEST(SceneCropperTest, CropFramesChecksCropSizePositive) {
  scene_summary.set_crop_window_width(-1);
  SceneCropper scene_cropper;
  std::vector<cv::Mat> cropped_frames;
+  std::vector<cv::Rect> crop_from_locations;
+  const auto& scene_frames = GetDefaultSceneFrames();
  const auto status = scene_cropper.CropFrames(
-      scene_summary, GetDefaultSceneFrames(), GetDefaultFocusPointFrames(),
-      GetFocusPointFrames(0), &cropped_frames);
+      scene_summary, scene_frames.size(), scene_frames,
+      GetDefaultFocusPointFrames(), GetFocusPointFrames(0), 0, 0,
+      &crop_from_locations, &cropped_frames);
  EXPECT_FALSE(status.ok());
  EXPECT_THAT(status.ToString(), HasSubstr("Crop width is non-positive."));
 }
@@ -124,9 +121,12 @@ TEST(SceneCropperTest, InitializesRetargeterChecksCropSizeNotExceedFrameSize) {
  scene_summary.set_crop_window_height(kSceneHeight + 1);
  SceneCropper scene_cropper;
  std::vector<cv::Mat> cropped_frames;
+  std::vector<cv::Rect> crop_from_locations;
+  const auto& scene_frames = GetDefaultSceneFrames();
  const auto status = scene_cropper.CropFrames(
-      scene_summary, GetDefaultSceneFrames(), GetDefaultFocusPointFrames(),
-      GetFocusPointFrames(0), &cropped_frames);
+      scene_summary, scene_frames.size(), scene_frames,
+      GetDefaultFocusPointFrames(), GetFocusPointFrames(0), 0, 0,
+      &crop_from_locations, &cropped_frames);
  EXPECT_FALSE(status.ok());
  EXPECT_THAT(status.ToString(),
              HasSubstr("Crop height exceeds frame height."));
@@ -136,9 +136,12 @@ TEST(SceneCropperTest, InitializesRetargeterChecksCropSizeNotExceedFrameSize) {
 TEST(SceneCropperTest, CropFramesWorksWithoutPriorFocusPointFrames) {
  SceneCropper scene_cropper;
  std::vector<cv::Mat> cropped_frames;
+  std::vector<cv::Rect> crop_from_locations;
+  const auto& scene_frames = GetDefaultSceneFrames();
  MP_ASSERT_OK(scene_cropper.CropFrames(
-      GetDefaultSceneKeyFrameCropSummary(), GetDefaultSceneFrames(),
-      GetDefaultFocusPointFrames(), GetFocusPointFrames(0), &cropped_frames));
+      GetDefaultSceneKeyFrameCropSummary(), scene_frames.size(), scene_frames,
+      GetDefaultFocusPointFrames(), GetFocusPointFrames(0), 0, 0,
+      &crop_from_locations, &cropped_frames));
  ASSERT_EQ(cropped_frames.size(), kNumSceneFrames);
  for (int i = 0; i < kNumSceneFrames; ++i) {
    EXPECT_EQ(cropped_frames[i].rows, kCropHeight);
@@ -150,9 +153,12 @@ TEST(SceneCropperTest, CropFramesWorksWithoutPriorFocusPointFrames) {
 TEST(SceneCropperTest, CropFramesWorksWithPriorFocusPointFrames) {
  SceneCropper scene_cropper;
  std::vector<cv::Mat> cropped_frames;
+  std::vector<cv::Rect> crop_from_locations;
+  const auto& scene_frames = GetDefaultSceneFrames();
  MP_EXPECT_OK(scene_cropper.CropFrames(
-      GetDefaultSceneKeyFrameCropSummary(), GetDefaultSceneFrames(),
-      GetDefaultFocusPointFrames(), GetFocusPointFrames(3), &cropped_frames));
+      GetDefaultSceneKeyFrameCropSummary(), scene_frames.size(), scene_frames,
+      GetDefaultFocusPointFrames(), GetFocusPointFrames(3), 0, 0,
+      &crop_from_locations, &cropped_frames));
  EXPECT_EQ(cropped_frames.size(), kNumSceneFrames);
  for (int i = 0; i < kNumSceneFrames; ++i) {
    EXPECT_EQ(cropped_frames[i].rows, kCropHeight);
@@ -0,0 +1,42 @@
+# Copyright 2019 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+licenses(["notice"])  # Apache 2.0
+
+package(default_visibility = ["//mediapipe/examples:__subpackages__"])
+
+cc_binary(
+    name = "face_mesh_tflite",
+    deps = [
+        "//mediapipe/examples/desktop:simple_run_graph_main",
+        "//mediapipe/graphs/face_mesh:desktop_calculators",
+    ],
+)
+
+cc_binary(
+    name = "face_mesh_cpu",
+    deps = [
+        "//mediapipe/examples/desktop:demo_run_graph_main",
+        "//mediapipe/graphs/face_mesh:desktop_live_calculators",
+    ],
+)
+
+# Linux only
+cc_binary(
+    name = "face_mesh_gpu",
+    deps = [
+        "//mediapipe/examples/desktop:demo_run_graph_main_gpu",
+        "//mediapipe/graphs/face_mesh:desktop_live_gpu_calculators",
+    ],
+)