Project import generated by Copybara.

GitOrigin-RevId: 0517756260533d374df93679965ca662d0ec6943

mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectioncpu/BUILD

@@ -62,7 +62,6 @@ android_library(
         "//third_party:androidx_appcompat",
         "//third_party:androidx_constraint_layout",
         "//third_party:androidx_legacy_support_v4",
-        "//third_party:androidx_material",
         "//third_party:androidx_recyclerview",
         "//third_party:opencv",
         "@androidx_concurrent_futures//jar",

mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectiongpu/BUILD

@@ -62,7 +62,6 @@ android_library(
         "//third_party:androidx_appcompat",
         "//third_party:androidx_constraint_layout",
         "//third_party:androidx_legacy_support_v4",
-        "//third_party:androidx_material",
         "//third_party:androidx_recyclerview",
         "//third_party:opencv",
         "@androidx_concurrent_futures//jar",

mediapipe/examples/android/src/java/com/google/mediapipe/apps/hairsegmentationgpu/BUILD

@@ -61,7 +61,6 @@ android_library(
         "//third_party:androidx_appcompat",
         "//third_party:androidx_constraint_layout",
         "//third_party:androidx_legacy_support_v4",
-        "//third_party:androidx_material",
         "//third_party:androidx_recyclerview",
         "//third_party:opencv",
         "@androidx_concurrent_futures//jar",

mediapipe/examples/android/src/java/com/google/mediapipe/apps/handdetectiongpu/BUILD

@@ -62,7 +62,6 @@ android_library(
         "//third_party:androidx_appcompat",
         "//third_party:androidx_constraint_layout",
         "//third_party:androidx_legacy_support_v4",
-        "//third_party:androidx_material",
         "//third_party:androidx_recyclerview",
         "//third_party:opencv",
         "@androidx_concurrent_futures//jar",

mediapipe/examples/android/src/java/com/google/mediapipe/apps/handtrackinggpu/BUILD

@@ -83,7 +83,6 @@ android_library(
         "//third_party:androidx_appcompat",
         "//third_party:androidx_constraint_layout",
         "//third_party:androidx_legacy_support_v4",
-        "//third_party:androidx_material",
         "//third_party:androidx_recyclerview",
         "//third_party:opencv",
         "@androidx_concurrent_futures//jar",

mediapipe/examples/android/src/java/com/google/mediapipe/apps/multihandtrackinggpu/BUILD

@@ -83,7 +83,6 @@ android_library(
         "//third_party:androidx_appcompat",
         "//third_party:androidx_constraint_layout",
         "//third_party:androidx_legacy_support_v4",
-        "//third_party:androidx_material",
         "//third_party:androidx_recyclerview",
         "//third_party:opencv",
         "@androidx_concurrent_futures//jar",

mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetectioncpu/BUILD

@@ -62,7 +62,6 @@ android_library(
         "//third_party:androidx_appcompat",
         "//third_party:androidx_constraint_layout",
         "//third_party:androidx_legacy_support_v4",
-        "//third_party:androidx_material",
         "//third_party:androidx_recyclerview",
         "//third_party:opencv",
         "@androidx_concurrent_futures//jar",

mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetectiongpu/BUILD

@@ -55,6 +55,7 @@ android_library(
     resource_files = glob(["res/**"]),
     deps = [
         ":mediapipe_jni_lib",
+        "//mediapipe/framework/formats:detection_java_proto_lite",
         "//mediapipe/java/com/google/mediapipe/components:android_camerax_helper",
         "//mediapipe/java/com/google/mediapipe/components:android_components",
         "//mediapipe/java/com/google/mediapipe/framework:android_framework",
@@ -62,7 +63,6 @@ android_library(
         "//third_party:androidx_appcompat",
         "//third_party:androidx_constraint_layout",
         "//third_party:androidx_legacy_support_v4",
-        "//third_party:androidx_material",
         "//third_party:androidx_recyclerview",
         "//third_party:opencv",
         "@androidx_concurrent_futures//jar",

mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetectiongpu/MainActivity.java

@@ -17,18 +17,22 @@ package com.google.mediapipe.apps.objectdetectiongpu;
 import android.graphics.SurfaceTexture;
 import android.os.Bundle;
 import androidx.appcompat.app.AppCompatActivity;
+import android.util.Log;
 import android.util.Size;
 import android.view.SurfaceHolder;
 import android.view.SurfaceView;
 import android.view.View;
 import android.view.ViewGroup;
+import com.google.mediapipe.formats.proto.DetectionProto.Detection;
 import com.google.mediapipe.components.CameraHelper;
 import com.google.mediapipe.components.CameraXPreviewHelper;
 import com.google.mediapipe.components.ExternalTextureConverter;
 import com.google.mediapipe.components.FrameProcessor;
 import com.google.mediapipe.components.PermissionHelper;
 import com.google.mediapipe.framework.AndroidAssetUtil;
+import com.google.mediapipe.framework.PacketGetter;
 import com.google.mediapipe.glutil.EglManager;
+import java.util.List;
 
 /** Main activity of MediaPipe example apps. */
 public class MainActivity extends AppCompatActivity {
@@ -37,6 +41,7 @@ public class MainActivity extends AppCompatActivity {
   private static final String BINARY_GRAPH_NAME = "objectdetectiongpu.binarypb";
   private static final String INPUT_VIDEO_STREAM_NAME = "input_video";
   private static final String OUTPUT_VIDEO_STREAM_NAME = "output_video";
+  private static final String OUTPUT_DETECTIONS_STREAM_NAME = "output_detections";
   private static final CameraHelper.CameraFacing CAMERA_FACING = CameraHelper.CameraFacing.BACK;
 
   // Flips the camera-preview frames vertically before sending them into FrameProcessor to be
@@ -90,6 +95,14 @@ public class MainActivity extends AppCompatActivity {
             OUTPUT_VIDEO_STREAM_NAME);
     processor.getVideoSurfaceOutput().setFlipY(FLIP_FRAMES_VERTICALLY);
 
+    processor.addPacketCallback(
+        OUTPUT_DETECTIONS_STREAM_NAME,
+        (packet) -> {
+          Log.d(TAG, "Received detections packet.");
+          List<Detection> detections = PacketGetter.getProtoVector(packet, Detection.parser());
+          Log.d(TAG, "[TS:" + packet.getTimestamp() + "] " + getDetectionsDebugString(detections));
+        });
+
     PermissionHelper.checkAndRequestCameraPermissions(this);
   }
 
@@ -164,4 +177,22 @@ public class MainActivity extends AppCompatActivity {
         });
     cameraHelper.startCamera(this, CAMERA_FACING, /*surfaceTexture=*/ null);
   }
+
+  private static String getDetectionsDebugString(List<Detection> detections) {
+    if (detections.isEmpty()) {
+      return "No detections";
+    }
+    String detectionsStr = "Number of objects detected: " + detections.size() + "\n";
+    int objectIndex = 0;
+    for (Detection detection : detections) {
+      detectionsStr += "\t#Object[" + objectIndex + "]: \n";
+      List<String> labels = detection.getLabelList();
+      List<Float> scores = detection.getScoreList();
+      for (int i = 0; i < labels.size(); ++i) {
+        detectionsStr += "\t\tLabel [" + i + "]: " + labels.get(i) + ", " + scores.get(i) + "\n";
+      }
+      ++objectIndex;
+    }
+    return detectionsStr;
+  }
 }

mediapipe/examples/coral/README.md

@@ -1,8 +1,12 @@
 # Coral Dev Board Setup (experimental)
 
-**Dislaimer**: Running MediaPipe on Coral is experimental, and this process may not be exact and is subject to change. These instructions have only been tested on the coral dev board with OS version _mendel day_, and may vary for different devices and workstations.
+**Dislaimer**: Running MediaPipe on Coral is experimental, and this process may
+not be exact and is subject to change. These instructions have only been tested
+on the [Coral Dev Board](https://coral.ai/products/dev-board/) with Mendel 4.0,
+and may vary for different devices and workstations.
 
-This file describes how to prepare a Google Coral Dev Board and setup a linux Docker container for building MediaPipe applications that run on Edge TPU.
+This file describes how to prepare a Coral Dev Board and setup a Linux
+Docker container for building MediaPipe applications that run on Edge TPU.
 
 ## Before creating the Docker
 

mediapipe/examples/coral/demo_run_graph_main.cc

@@ -13,6 +13,7 @@
 // limitations under the License.
 //
 // An example of sending OpenCV webcam frames into a MediaPipe graph.
+#include <cstdlib>
 
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/formats/image_frame.h"
@@ -89,7 +90,6 @@ DEFINE_string(output_video_path, "",
   MP_RETURN_IF_ERROR(graph.StartRun({}));
 
   LOG(INFO) << "Start grabbing and processing frames.";
-  size_t frame_timestamp = 0;
   bool grab_frames = true;
   while (grab_frames) {
     // Capture opencv camera or video frame.
@@ -110,9 +110,11 @@ DEFINE_string(output_video_path, "",
     camera_frame.copyTo(input_frame_mat);
 
     // Send image packet into the graph.
+    size_t frame_timestamp_us =
+        (double)cv::getTickCount() / (double)cv::getTickFrequency() * 1e6;
     MP_RETURN_IF_ERROR(graph.AddPacketToInputStream(
         kInputStream, mediapipe::Adopt(input_frame.release())
-                          .At(mediapipe::Timestamp(frame_timestamp++))));
+                          .At(mediapipe::Timestamp(frame_timestamp_us))));
 
     // Get the graph result packet, or stop if that fails.
     mediapipe::Packet packet;
@@ -144,8 +146,9 @@ int main(int argc, char** argv) {
   ::mediapipe::Status run_status = RunMPPGraph();
   if (!run_status.ok()) {
     LOG(ERROR) << "Failed to run the graph: " << run_status.message();
+    return EXIT_FAILURE;
   } else {
     LOG(INFO) << "Success!";
   }
-  return 0;
+  return EXIT_SUCCESS;
 }

mediapipe/examples/desktop/autoflip/BUILD

@@ -0,0 +1,56 @@
+load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
+
+# Copyright 2019 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+licenses(["notice"])  # Apache 2.0
+
+package(default_visibility = ["//mediapipe/examples:__subpackages__"])
+
+proto_library(
+    name = "autoflip_messages_proto",
+    srcs = ["autoflip_messages.proto"],
+    deps = [
+        "//mediapipe/framework:calculator_proto",
+    ],
+)
+
+mediapipe_cc_proto_library(
+    name = "autoflip_messages_cc_proto",
+    srcs = ["autoflip_messages.proto"],
+    cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
+    visibility = ["//mediapipe/examples:__subpackages__"],
+    deps = [":autoflip_messages_proto"],
+)
+
+cc_binary(
+    name = "run_autoflip",
+    deps = [
+        "//mediapipe/calculators/core:packet_thinner_calculator",
+        "//mediapipe/calculators/image:scale_image_calculator",
+        "//mediapipe/calculators/video:opencv_video_decoder_calculator",
+        "//mediapipe/calculators/video:opencv_video_encoder_calculator",
+        "//mediapipe/calculators/video:video_pre_stream_calculator",
+        "//mediapipe/examples/desktop:simple_run_graph_main",
+        "//mediapipe/examples/desktop/autoflip/calculators:border_detection_calculator",
+        "//mediapipe/examples/desktop/autoflip/calculators:face_to_region_calculator",
+        "//mediapipe/examples/desktop/autoflip/calculators:localization_to_region_calculator",
+        "//mediapipe/examples/desktop/autoflip/calculators:scene_cropping_calculator",
+        "//mediapipe/examples/desktop/autoflip/calculators:shot_boundary_calculator",
+        "//mediapipe/examples/desktop/autoflip/calculators:signal_fusing_calculator",
+        "//mediapipe/examples/desktop/autoflip/calculators:video_filtering_calculator",
+        "//mediapipe/examples/desktop/autoflip/subgraph:autoflip_face_detection_subgraph",
+        "//mediapipe/examples/desktop/autoflip/subgraph:autoflip_object_detection_subgraph",
+    ],
+)

mediapipe/examples/desktop/autoflip/README.md

@@ -0,0 +1,25 @@
+### Steps to run the AutoFlip video cropping graph
+
+1.  Checkout the repository and follow
+    [the installation instructions](https://github.com/google/mediapipe/blob/master/mediapipe/docs/install.md)
+    to set up MediaPipe.
+
+    ```bash
+    git clone https://github.com/google/mediapipe.git
+    cd mediapipe
+    ```
+
+2.  Build and run the run_autoflip binary to process a local video.
+
+    ```bash
+    bazel build -c opt --define MEDIAPIPE_DISABLE_GPU=1 \
+      mediapipe/examples/desktop/autoflip:run_autoflip
+
+    GLOG_logtostderr=1 bazel-bin/mediapipe/examples/desktop/autoflip/run_autoflip \
+      --calculator_graph_config_file=mediapipe/examples/desktop/autoflip/autoflip_graph.pbtxt \
+      --input_side_packets=input_video_path=/absolute/path/to/the/local/video/file,\
+        output_video_path=/absolute/path/to/save/the/output/video/file,\
+        aspect_ratio=width:height
+    ```
+
+3.  View the cropped video.

mediapipe/examples/desktop/autoflip/autoflip_graph.pbtxt

@@ -0,0 +1,202 @@
+# Autoflip graph that only renders the final cropped video. For use with
+# end user applications.
+max_queue_size: -1
+
+# VIDEO_PREP: Decodes an input video file into images and a video header.
+node {
+  calculator: "OpenCvVideoDecoderCalculator"
+  input_side_packet: "INPUT_FILE_PATH:input_video_path"
+  output_stream: "VIDEO:video_raw"
+  output_stream: "VIDEO_PRESTREAM:video_header"
+  output_side_packet: "SAVED_AUDIO_PATH:audio_path"
+}
+
+# VIDEO_PREP: Scale the input video before feature extraction.
+node {
+  calculator: "ScaleImageCalculator"
+  input_stream: "FRAMES:video_raw"
+  input_stream: "VIDEO_HEADER:video_header"
+  output_stream: "FRAMES:video_frames_scaled"
+  options: {
+    [mediapipe.ScaleImageCalculatorOptions.ext]: {
+      preserve_aspect_ratio: true
+      output_format: SRGB
+      target_width: 480
+      algorithm: DEFAULT_WITHOUT_UPSCALE
+    }
+  }
+}
+
+# VIDEO_PREP: Create a low frame rate stream for feature extraction.
+node {
+  calculator: "PacketThinnerCalculator"
+  input_stream: "video_frames_scaled"
+  output_stream: "video_frames_scaled_downsampled"
+  options: {
+    [mediapipe.PacketThinnerCalculatorOptions.ext]: {
+      thinner_type: ASYNC
+      period: 500000
+    }
+  }
+}
+
+# DETECTION: find borders around the video and major background color.
+node {
+  calculator: "BorderDetectionCalculator"
+  input_stream: "VIDEO:video_raw"
+  output_stream: "DETECTED_BORDERS:borders"
+}
+
+# DETECTION: find shot/scene boundaries on the full frame rate stream.
+node {
+  calculator: "ShotBoundaryCalculator"
+  input_stream: "VIDEO:video_frames_scaled"
+  output_stream: "IS_SHOT_CHANGE:shot_change"
+  options {
+    [mediapipe.autoflip.ShotBoundaryCalculatorOptions.ext] {
+      min_shot_span: 0.2
+      min_motion: 0.3
+      window_size: 15
+      min_shot_measure: 10
+      min_motion_with_shot_measure: 0.05
+    }
+  }
+}
+
+# DETECTION: find faces on the down sampled stream
+node {
+  calculator: "AutoFlipFaceDetectionSubgraph"
+  input_stream: "VIDEO:video_frames_scaled_downsampled"
+  output_stream: "DETECTIONS:face_detections"
+}
+node {
+  calculator: "FaceToRegionCalculator"
+  input_stream: "VIDEO:video_frames_scaled_downsampled"
+  input_stream: "FACES:face_detections"
+  output_stream: "REGIONS:face_regions"
+}
+
+# DETECTION: find objects on the down sampled stream
+node {
+  calculator: "AutoFlipObjectDetectionSubgraph"
+  input_stream: "VIDEO:video_frames_scaled_downsampled"
+  output_stream: "DETECTIONS:object_detections"
+}
+node {
+  calculator: "LocalizationToRegionCalculator"
+  input_stream: "DETECTIONS:object_detections"
+  output_stream: "REGIONS:object_regions"
+  options {
+    [mediapipe.autoflip.LocalizationToRegionCalculatorOptions.ext] {
+      output_all_signals: true
+    }
+  }
+}
+
+# SIGNAL FUSION: Combine detections (with weights) on each frame
+node {
+  calculator: "SignalFusingCalculator"
+  input_stream: "shot_change"
+  input_stream: "face_regions"
+  input_stream: "object_regions"
+  output_stream: "salient_regions"
+  options {
+    [mediapipe.autoflip.SignalFusingCalculatorOptions.ext] {
+      signal_settings {
+        type { standard: FACE_CORE_LANDMARKS }
+        min_score: 0.85
+        max_score: 0.9
+        is_required: false
+      }
+      signal_settings {
+        type { standard: FACE_ALL_LANDMARKS }
+        min_score: 0.8
+        max_score: 0.85
+        is_required: false
+      }
+      signal_settings {
+        type { standard: FACE_FULL }
+        min_score: 0.8
+        max_score: 0.85
+        is_required: false
+      }
+      signal_settings {
+        type: { standard: HUMAN }
+        min_score: 0.75
+        max_score: 0.8
+        is_required: false
+      }
+      signal_settings {
+        type: { standard: PET }
+        min_score: 0.7
+        max_score: 0.75
+        is_required: false
+      }
+      signal_settings {
+        type: { standard: CAR }
+        min_score: 0.7
+        max_score: 0.75
+        is_required: false
+      }
+      signal_settings {
+        type: { standard: OBJECT }
+        min_score: 0.1
+        max_score: 0.2
+        is_required: false
+      }
+    }
+  }
+}
+
+# CROPPING: make decisions about how to crop each frame.
+node {
+  calculator: "SceneCroppingCalculator"
+  input_side_packet: "EXTERNAL_ASPECT_RATIO:aspect_ratio"
+  input_stream: "VIDEO_FRAMES:video_raw"
+  input_stream: "KEY_FRAMES:video_frames_scaled_downsampled"
+  input_stream: "DETECTION_FEATURES:salient_regions"
+  input_stream: "STATIC_FEATURES:borders"
+  input_stream: "SHOT_BOUNDARIES:shot_change"
+  output_stream: "CROPPED_FRAMES:cropped_frames"
+  options: {
+    [mediapipe.autoflip.SceneCroppingCalculatorOptions.ext]: {
+      max_scene_size: 600
+      key_frame_crop_options: {
+        score_aggregation_type: CONSTANT
+      }
+      scene_camera_motion_analyzer_options: {
+        motion_stabilization_threshold_percent: 0.3
+        salient_point_bound: 0.499
+      }
+      padding_parameters: {
+        blur_cv_size: 200
+        overlay_opacity: 0.6
+      }
+      target_size_type: MAXIMIZE_TARGET_DIMENSION
+    }
+  }
+}
+
+# ENCODING(required): encode the video stream for the final cropped output.
+node {
+  calculator: "VideoPreStreamCalculator"
+  # Fetch frame format and dimension from input frames.
+  input_stream: "FRAME:cropped_frames"
+  # Copying frame rate and duration from original video.
+  input_stream: "VIDEO_PRESTREAM:video_header"
+  output_stream: "output_frames_video_header"
+}
+
+node {
+  calculator: "OpenCvVideoEncoderCalculator"
+  input_stream: "VIDEO:cropped_frames"
+  input_stream: "VIDEO_PRESTREAM:output_frames_video_header"
+  input_side_packet: "OUTPUT_FILE_PATH:output_video_path"
+  input_side_packet: "AUDIO_FILE_PATH:audio_path"
+  options: {
+    [mediapipe.OpenCvVideoEncoderCalculatorOptions.ext]: {
+      codec: "avc1"
+      video_format: "mp4"
+    }
+  }
+}

mediapipe/examples/desktop/autoflip/autoflip_graph_development.pbtxt

@@ -0,0 +1,252 @@
+# Autoflip graph that renders the final cropped video and debugging videos.
+# For use by developers who may be adding signals and adjusting weights.
+max_queue_size: -1
+
+# VIDEO_PREP: Decodes an input video file into images and a video header.
+node {
+  calculator: "OpenCvVideoDecoderCalculator"
+  input_side_packet: "INPUT_FILE_PATH:input_video_path"
+  output_stream: "VIDEO:video_raw"
+  output_stream: "VIDEO_PRESTREAM:video_header"
+  output_side_packet: "SAVED_AUDIO_PATH:audio_path"
+}
+
+# VIDEO_PREP: Scale the input video before feature extraction.
+node {
+  calculator: "ScaleImageCalculator"
+  input_stream: "FRAMES:video_raw"
+  input_stream: "VIDEO_HEADER:video_header"
+  output_stream: "FRAMES:video_frames_scaled"
+  options: {
+    [mediapipe.ScaleImageCalculatorOptions.ext]: {
+      preserve_aspect_ratio: true
+      output_format: SRGB
+      target_width: 480
+      algorithm: DEFAULT_WITHOUT_UPSCALE
+    }
+  }
+}
+
+# VIDEO_PREP: Create a low frame rate stream for feature extraction.
+node {
+  calculator: "PacketThinnerCalculator"
+  input_stream: "video_frames_scaled"
+  output_stream: "video_frames_scaled_downsampled"
+  options: {
+    [mediapipe.PacketThinnerCalculatorOptions.ext]: {
+      thinner_type: ASYNC
+      period: 500000
+    }
+  }
+}
+
+# DETECTION: find borders around the video and major background color.
+node {
+  calculator: "BorderDetectionCalculator"
+  input_stream: "VIDEO:video_raw"
+  output_stream: "DETECTED_BORDERS:borders"
+}
+
+# DETECTION: find shot/scene boundaries on the full frame rate stream.
+node {
+  calculator: "ShotBoundaryCalculator"
+  input_stream: "VIDEO:video_frames_scaled"
+  output_stream: "IS_SHOT_CHANGE:shot_change"
+  options {
+    [mediapipe.autoflip.ShotBoundaryCalculatorOptions.ext] {
+      min_shot_span: 0.2
+      min_motion: 0.3
+      window_size: 15
+      min_shot_measure: 10
+      min_motion_with_shot_measure: 0.05
+    }
+  }
+}
+
+# DETECTION: find faces on the down sampled stream
+node {
+  calculator: "AutoFlipFaceDetectionSubgraph"
+  input_stream: "VIDEO:video_frames_scaled_downsampled"
+  output_stream: "DETECTIONS:face_detections"
+}
+node {
+  calculator: "FaceToRegionCalculator"
+  input_stream: "VIDEO:video_frames_scaled_downsampled"
+  input_stream: "FACES:face_detections"
+  output_stream: "REGIONS:face_regions"
+}
+
+# DETECTION: find objects on the down sampled stream
+node {
+  calculator: "AutoFlipObjectDetectionSubgraph"
+  input_stream: "VIDEO:video_frames_scaled_downsampled"
+  output_stream: "DETECTIONS:object_detections"
+}
+node {
+  calculator: "LocalizationToRegionCalculator"
+  input_stream: "DETECTIONS:object_detections"
+  output_stream: "REGIONS:object_regions"
+  options {
+    [mediapipe.autoflip.LocalizationToRegionCalculatorOptions.ext] {
+      output_all_signals: true
+    }
+  }
+}
+
+# SIGNAL FUSION: Combine detections (with weights) on each frame
+node {
+  calculator: "SignalFusingCalculator"
+  input_stream: "shot_change"
+  input_stream: "face_regions"
+  input_stream: "object_regions"
+  output_stream: "salient_regions"
+  options {
+    [mediapipe.autoflip.SignalFusingCalculatorOptions.ext] {
+      signal_settings {
+        type { standard: FACE_CORE_LANDMARKS }
+        min_score: 0.85
+        max_score: 0.9
+        is_required: false
+      }
+      signal_settings {
+        type { standard: FACE_ALL_LANDMARKS }
+        min_score: 0.8
+        max_score: 0.85
+        is_required: false
+      }
+      signal_settings {
+        type { standard: FACE_FULL }
+        min_score: 0.8
+        max_score: 0.85
+        is_required: false
+      }
+      signal_settings {
+        type: { standard: HUMAN }
+        min_score: 0.75
+        max_score: 0.8
+        is_required: false
+      }
+      signal_settings {
+        type: { standard: PET }
+        min_score: 0.7
+        max_score: 0.75
+        is_required: false
+      }
+      signal_settings {
+        type: { standard: CAR }
+        min_score: 0.7
+        max_score: 0.75
+        is_required: false
+      }
+      signal_settings {
+        type: { standard: OBJECT }
+        min_score: 0.1
+        max_score: 0.2
+        is_required: false
+      }
+    }
+  }
+}
+
+# CROPPING: make decisions about how to crop each frame.
+node {
+  calculator: "SceneCroppingCalculator"
+  input_side_packet: "EXTERNAL_ASPECT_RATIO:aspect_ratio"
+  input_stream: "VIDEO_FRAMES:video_raw"
+  input_stream: "KEY_FRAMES:video_frames_scaled_downsampled"
+  input_stream: "DETECTION_FEATURES:salient_regions"
+  input_stream: "STATIC_FEATURES:borders"
+  input_stream: "SHOT_BOUNDARIES:shot_change"
+  output_stream: "CROPPED_FRAMES:cropped_frames"
+  output_stream: "KEY_FRAME_CROP_REGION_VIZ_FRAMES:key_frame_crop_viz_frames"
+  output_stream: "SALIENT_POINT_FRAME_VIZ_FRAMES:salient_point_viz_frames"
+  options: {
+    [mediapipe.autoflip.SceneCroppingCalculatorOptions.ext]: {
+      max_scene_size: 600
+      key_frame_crop_options: {
+        score_aggregation_type: CONSTANT
+      }
+      scene_camera_motion_analyzer_options: {
+        motion_stabilization_threshold_percent: 0.3
+        salient_point_bound: 0.499
+      }
+      padding_parameters: {
+        blur_cv_size: 200
+        overlay_opacity: 0.6
+      }
+      target_size_type: MAXIMIZE_TARGET_DIMENSION
+    }
+  }
+}
+
+# ENCODING(required): encode the video stream for the final cropped output.
+node {
+  calculator: "VideoPreStreamCalculator"
+  # Fetch frame format and dimension from input frames.
+  input_stream: "FRAME:cropped_frames"
+  # Copying frame rate and duration from original video.
+  input_stream: "VIDEO_PRESTREAM:video_header"
+  output_stream: "output_frames_video_header"
+}
+
+node {
+  calculator: "OpenCvVideoEncoderCalculator"
+  input_stream: "VIDEO:cropped_frames"
+  input_stream: "VIDEO_PRESTREAM:output_frames_video_header"
+  input_side_packet: "OUTPUT_FILE_PATH:output_video_path"
+  input_side_packet: "AUDIO_FILE_PATH:audio_path"
+  options: {
+    [mediapipe.OpenCvVideoEncoderCalculatorOptions.ext]: {
+      codec: "avc1"
+      video_format: "mp4"
+    }
+  }
+}
+
+# ENCODING(optional): encode the video stream for the key_frame_crop_viz_frames
+# output.  Draws boxes around required and non-required objects.
+node {
+  calculator: "VideoPreStreamCalculator"
+  # Fetch frame format and dimension from input frames.
+  input_stream: "FRAME:key_frame_crop_viz_frames"
+  # Copying frame rate and duration from original video.
+  input_stream: "VIDEO_PRESTREAM:video_header"
+  output_stream: "key_frame_crop_viz_frames_header"
+}
+
+node {
+  calculator: "OpenCvVideoEncoderCalculator"
+  input_stream: "VIDEO:key_frame_crop_viz_frames"
+  input_stream: "VIDEO_PRESTREAM:key_frame_crop_viz_frames_header"
+  input_side_packet: "OUTPUT_FILE_PATH:key_frame_crop_viz_frames_path"
+  options: {
+    [mediapipe.OpenCvVideoEncoderCalculatorOptions.ext]: {
+      codec: "avc1"
+      video_format: "mp4"
+    }
+  }
+}
+
+# ENCODING(optional): encode the video stream for the salient_point_viz_frames
+# output. Draws the focus points and the scene crop window (red).
+node {
+  calculator: "VideoPreStreamCalculator"
+  # Fetch frame format and dimension from input frames.
+  input_stream: "FRAME:salient_point_viz_frames"
+  # Copying frame rate and duration from original video.
+  input_stream: "VIDEO_PRESTREAM:video_header"
+  output_stream: "salient_point_viz_frames_header"
+}
+
+node {
+  calculator: "OpenCvVideoEncoderCalculator"
+  input_stream: "VIDEO:salient_point_viz_frames"
+  input_stream: "VIDEO_PRESTREAM:salient_point_viz_frames_header"
+  input_side_packet: "OUTPUT_FILE_PATH:salient_point_viz_frames_path"
+  options: {
+    [mediapipe.OpenCvVideoEncoderCalculatorOptions.ext]: {
+      codec: "avc1"
+      video_format: "mp4"
+    }
+  }
+}

mediapipe/examples/desktop/autoflip/autoflip_messages.proto

@@ -0,0 +1,153 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Proto messages used for the AutoFlip Pipeline.
+syntax = "proto2";
+
+package mediapipe.autoflip;
+
+import "mediapipe/framework/calculator.proto";
+
+// Borders detected on the frame as well as non-border color (if present).
+// Next tag: 4
+message StaticFeatures {
+  // A list of the static parts for a frame.
+  repeated Border border = 1;
+  // The background color (only set if solid color).
+  optional Color solid_background = 2;
+  // Area of the image that is not a border.
+  optional Rect non_static_area = 3;
+}
+
+// A static border area within the video.
+// Next tag: 3
+message Border {
+  // Original location within the input frame.
+  optional Rect border_position = 1;
+  // Position for static area.
+  // Next tag: 3
+  enum RelativePosition {
+    TOP = 1;
+    BOTTOM = 2;
+  }
+  // Top or bottom position.
+  optional RelativePosition relative_position = 2;
+}
+
+// Rectangle (opencv format).
+// Next tag: 5
+message Rect {
+  optional int32 x = 1;
+  optional int32 y = 2;
+  optional int32 width = 3;
+  optional int32 height = 4;
+}
+
+// Color (RGB 8bit)
+// Next tag: 4
+message Color {
+  optional int32 r = 1;
+  optional int32 g = 2;
+  optional int32 b = 3;
+}
+
+// Rectangle (opencv format).
+// Next tag: 5
+message RectF {
+  optional float x = 1;
+  optional float y = 2;
+  optional float width = 3;
+  optional float height = 4;
+}
+
+// An image region of interest (eg a detected face or object), accompanied by an
+// importance score.
+// Next tag: 9
+message SalientRegion {
+  reserved 3;
+  // The bounding box for this region in the image.
+  optional Rect location = 1;
+
+  // The bounding box for this region in the image normalized.
+  optional RectF location_normalized = 8;
+
+  // A score indicating the importance of this region.
+  optional float score = 2;
+
+  // A tracking id used to identify this region across video frames. Not always
+  // set.
+  optional int64 tracking_id = 4;
+
+  // If true, this region is required to be present in the final video (eg it
+  // contains text that cannot be cropped).
+  optional bool is_required = 5 [default = false];
+
+  // Type of signal carried in this message.
+  optional SignalType signal_type = 6;
+
+  // If true, object cannot move in the output window (e.g. text would look
+  // strange moving around).
+  optional bool requires_static_location = 7 [default = false];
+}
+
+// Stores the message type, including standard types (face, object) and custom
+// types defined by a string id.
+// Next tag: 3
+message SignalType {
+  enum StandardType {
+    UNSET = 0;
+    // Full face bounding boxed detected.
+    FACE_FULL = 1;
+    // Face landmarks for eyes, nose, chin only.
+    FACE_CORE_LANDMARKS = 2;
+    // All face landmarks (eyes, ears, nose, chin).
+    FACE_ALL_LANDMARKS = 3;
+    // A specific face landmark.
+    FACE_LANDMARK = 4;
+    HUMAN = 5;
+    CAR = 6;
+    PET = 7;
+    OBJECT = 8;
+    MOTION = 9;
+    TEXT = 10;
+    LOGO = 11;
+    USER_HINT = 12;
+  }
+  oneof Signal {
+    StandardType standard = 1;
+    string custom = 2;
+  }
+}
+
+// Features extracted from a image.
+// Next tag: 3
+message DetectionSet {
+  // Mask image showing pixel-wise values at a given location.
+  optional string encoded_mask = 1;
+  // List of rectangle detections.
+  repeated SalientRegion detections = 2;
+}
+
+// General settings needed for multiple calculators.
+message ConversionOptions {
+  extend mediapipe.CalculatorOptions {
+    optional ConversionOptions ext = 284806832;
+  }
+  // Target output width of the conversion.
+  optional int32 target_width = 1;
+  // Target output height of the conversion.
+  optional int32 target_height = 2;
+}
+
+// TODO: Move other autoflip messages into this area.

mediapipe/examples/desktop/autoflip/calculators/BUILD

@@ -0,0 +1,426 @@
+load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
+
+# Copyright 2019 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+licenses(["notice"])  # Apache 2.0
+
+package(default_visibility = ["//mediapipe/examples:__subpackages__"])
+
+cc_library(
+    name = "border_detection_calculator",
+    srcs = ["border_detection_calculator.cc"],
+    deps = [
+        ":border_detection_calculator_cc_proto",
+        "//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:image_frame",
+        "//mediapipe/framework/formats:image_frame_opencv",
+        "//mediapipe/framework/port:opencv_core",
+        "//mediapipe/framework/port:opencv_imgproc",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+    ],
+    alwayslink = 1,
+)
+
+proto_library(
+    name = "border_detection_calculator_proto",
+    srcs = ["border_detection_calculator.proto"],
+    deps = [
+        "//mediapipe/framework:calculator_proto",
+    ],
+)
+
+mediapipe_cc_proto_library(
+    name = "border_detection_calculator_cc_proto",
+    srcs = ["border_detection_calculator.proto"],
+    cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
+    visibility = ["//mediapipe/examples:__subpackages__"],
+    deps = [":border_detection_calculator_proto"],
+)
+
+cc_test(
+    name = "border_detection_calculator_test",
+    srcs = [
+        "border_detection_calculator_test.cc",
+    ],
+    linkstatic = 1,
+    deps = [
+        ":border_detection_calculator",
+        ":border_detection_calculator_cc_proto",
+        "//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:calculator_runner",
+        "//mediapipe/framework/formats:image_frame",
+        "//mediapipe/framework/formats:image_frame_opencv",
+        "//mediapipe/framework/port:benchmark",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:opencv_core",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
+cc_library(
+    name = "video_filtering_calculator",
+    srcs = ["video_filtering_calculator.cc"],
+    copts = ["-fexceptions"],
+    features = ["-use_header_modules"],  # Incompatible with -fexceptions.
+    deps = [
+        ":video_filtering_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:image_frame",
+        "//mediapipe/framework/formats:image_frame_opencv",
+        "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/strings",
+    ],
+    alwayslink = 1,
+)
+
+proto_library(
+    name = "video_filtering_calculator_proto",
+    srcs = ["video_filtering_calculator.proto"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_proto",
+    ],
+)
+
+mediapipe_cc_proto_library(
+    name = "video_filtering_calculator_cc_proto",
+    srcs = ["video_filtering_calculator.proto"],
+    cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
+    visibility = ["//visibility:public"],
+    deps = [":video_filtering_calculator_proto"],
+)
+
+cc_test(
+    name = "video_filtering_calculator_test",
+    srcs = ["video_filtering_calculator_test.cc"],
+    deps = [
+        ":video_filtering_calculator",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:calculator_runner",
+        "//mediapipe/framework/formats:image_frame",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
+proto_library(
+    name = "scene_cropping_calculator_proto",
+    srcs = ["scene_cropping_calculator.proto"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/examples/desktop/autoflip/quality:cropping_proto",
+        "//mediapipe/framework:calculator_proto",
+    ],
+)
+
+mediapipe_cc_proto_library(
+    name = "scene_cropping_calculator_cc_proto",
+    srcs = ["scene_cropping_calculator.proto"],
+    cc_deps = [
+        "//mediapipe/examples/desktop/autoflip/quality:cropping_cc_proto",
+        "//mediapipe/framework:calculator_cc_proto",
+    ],
+    visibility = ["//visibility:public"],
+    deps = [":scene_cropping_calculator_proto"],
+)
+
+cc_library(
+    name = "scene_cropping_calculator",
+    srcs = ["scene_cropping_calculator.cc"],
+    hdrs = ["scene_cropping_calculator.h"],
+    deps = [
+        ":scene_cropping_calculator_cc_proto",
+        "//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto",
+        "//mediapipe/examples/desktop/autoflip/quality:cropping_cc_proto",
+        "//mediapipe/examples/desktop/autoflip/quality:focus_point_cc_proto",
+        "//mediapipe/examples/desktop/autoflip/quality:frame_crop_region_computer",
+        "//mediapipe/examples/desktop/autoflip/quality:padding_effect_generator",
+        "//mediapipe/examples/desktop/autoflip/quality:piecewise_linear_function",
+        "//mediapipe/examples/desktop/autoflip/quality:polynomial_regression_path_solver",
+        "//mediapipe/examples/desktop/autoflip/quality:scene_camera_motion_analyzer",
+        "//mediapipe/examples/desktop/autoflip/quality:scene_cropper",
+        "//mediapipe/examples/desktop/autoflip/quality:scene_cropping_viz",
+        "//mediapipe/examples/desktop/autoflip/quality:utils",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:timestamp",
+        "//mediapipe/framework/formats:image_frame",
+        "//mediapipe/framework/formats:image_frame_opencv",
+        "//mediapipe/framework/port:opencv_core",
+        "//mediapipe/framework/port:opencv_imgproc",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/memory",
+        "@com_google_absl//absl/strings:str_format",
+    ],
+    alwayslink = 1,  # buildozer: disable=alwayslink-with-hdrs
+)
+
+cc_test(
+    name = "scene_cropping_calculator_test",
+    size = "large",
+    timeout = "long",
+    srcs = ["scene_cropping_calculator_test.cc"],
+    deps = [
+        ":scene_cropping_calculator",
+        "//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:calculator_runner",
+        "//mediapipe/framework/formats:image_frame",
+        "//mediapipe/framework/formats:image_frame_opencv",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:opencv_core",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+    ],
+)
+
+cc_library(
+    name = "signal_fusing_calculator",
+    srcs = ["signal_fusing_calculator.cc"],
+    deps = [
+        ":signal_fusing_calculator_cc_proto",
+        "//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:image_frame",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+    ],
+    alwayslink = 1,
+)
+
+proto_library(
+    name = "signal_fusing_calculator_proto",
+    srcs = ["signal_fusing_calculator.proto"],
+    deps = [
+        "//mediapipe/examples/desktop/autoflip:autoflip_messages_proto",
+        "//mediapipe/framework:calculator_proto",
+    ],
+)
+
+mediapipe_cc_proto_library(
+    name = "signal_fusing_calculator_cc_proto",
+    srcs = ["signal_fusing_calculator.proto"],
+    cc_deps = [
+        "//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto",
+        "//mediapipe/framework:calculator_cc_proto",
+    ],
+    visibility = ["//mediapipe/examples:__subpackages__"],
+    deps = [":signal_fusing_calculator_proto"],
+)
+
+cc_test(
+    name = "signal_fusing_calculator_test",
+    srcs = ["signal_fusing_calculator_test.cc"],
+    linkstatic = 1,
+    deps = [
+        ":signal_fusing_calculator",
+        ":signal_fusing_calculator_cc_proto",
+        "//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:calculator_runner",
+        "//mediapipe/framework/formats:image_frame",
+        "//mediapipe/framework/formats:image_frame_opencv",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
+cc_library(
+    name = "shot_boundary_calculator",
+    srcs = ["shot_boundary_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":shot_boundary_calculator_cc_proto",
+        "//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:timestamp",
+        "//mediapipe/framework/formats:image_frame",
+        "//mediapipe/framework/formats:image_frame_opencv",
+        "//mediapipe/framework/port:opencv_imgproc",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+    ],
+    alwayslink = 1,
+)
+
+proto_library(
+    name = "shot_boundary_calculator_proto",
+    srcs = ["shot_boundary_calculator.proto"],
+    deps = [
+        "//mediapipe/examples/desktop/autoflip:autoflip_messages_proto",
+        "//mediapipe/framework:calculator_proto",
+    ],
+)
+
+mediapipe_cc_proto_library(
+    name = "shot_boundary_calculator_cc_proto",
+    srcs = ["shot_boundary_calculator.proto"],
+    cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
+    visibility = ["//mediapipe/examples:__subpackages__"],
+    deps = [":shot_boundary_calculator_proto"],
+)
+
+cc_test(
+    name = "shot_boundary_calculator_test",
+    srcs = ["shot_boundary_calculator_test.cc"],
+    data = ["//mediapipe/examples/desktop/autoflip/calculators/testdata:test_images"],
+    linkstatic = 1,
+    deps = [
+        ":shot_boundary_calculator",
+        ":shot_boundary_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:calculator_runner",
+        "//mediapipe/framework/deps:file_path",
+        "//mediapipe/framework/formats:image_frame",
+        "//mediapipe/framework/formats:image_frame_opencv",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:opencv_core",
+        "//mediapipe/framework/port:opencv_imgcodecs",
+        "//mediapipe/framework/port:opencv_imgproc",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
+cc_library(
+    name = "face_to_region_calculator",
+    srcs = ["face_to_region_calculator.cc"],
+    deps = [
+        ":face_to_region_calculator_cc_proto",
+        "//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto",
+        "//mediapipe/examples/desktop/autoflip/quality:visual_scorer",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:detection_cc_proto",
+        "//mediapipe/framework/formats:image_frame",
+        "//mediapipe/framework/formats:image_frame_opencv",
+        "//mediapipe/framework/formats:location_data_cc_proto",
+        "//mediapipe/framework/port:opencv_core",
+        "//mediapipe/framework/port:opencv_imgproc",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/memory",
+    ],
+    alwayslink = 1,
+)
+
+proto_library(
+    name = "face_to_region_calculator_proto",
+    srcs = ["face_to_region_calculator.proto"],
+    deps = [
+        "//mediapipe/examples/desktop/autoflip/quality:visual_scorer_proto",
+        "//mediapipe/framework:calculator_proto",
+    ],
+)
+
+mediapipe_cc_proto_library(
+    name = "face_to_region_calculator_cc_proto",
+    srcs = ["face_to_region_calculator.proto"],
+    cc_deps = [
+        "//mediapipe/examples/desktop/autoflip/quality:visual_scorer_cc_proto",
+        "//mediapipe/framework:calculator_cc_proto",
+    ],
+    visibility = ["//mediapipe/examples:__subpackages__"],
+    deps = [":face_to_region_calculator_proto"],
+)
+
+cc_test(
+    name = "face_to_region_calculator_test",
+    srcs = ["face_to_region_calculator_test.cc"],
+    linkstatic = 1,
+    deps = [
+        ":face_to_region_calculator",
+        ":face_to_region_calculator_cc_proto",
+        "//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:calculator_runner",
+        "//mediapipe/framework/formats:detection_cc_proto",
+        "//mediapipe/framework/formats:image_frame",
+        "//mediapipe/framework/formats:image_frame_opencv",
+        "//mediapipe/framework/formats:location_data_cc_proto",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
+proto_library(
+    name = "localization_to_region_calculator_proto",
+    srcs = ["localization_to_region_calculator.proto"],
+    deps = [
+        "//mediapipe/framework:calculator_proto",
+    ],
+)
+
+mediapipe_cc_proto_library(
+    name = "localization_to_region_calculator_cc_proto",
+    srcs = ["localization_to_region_calculator.proto"],
+    cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
+    visibility = ["//mediapipe/examples:__subpackages__"],
+    deps = [":localization_to_region_calculator_proto"],
+)
+
+cc_library(
+    name = "localization_to_region_calculator",
+    srcs = ["localization_to_region_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":localization_to_region_calculator_cc_proto",
+        "//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:detection_cc_proto",
+        "//mediapipe/framework/formats:location_data_cc_proto",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/memory",
+    ],
+    alwayslink = 1,
+)
+
+cc_test(
+    name = "localization_to_region_calculator_test",
+    srcs = ["localization_to_region_calculator_test.cc"],
+    linkstatic = 1,
+    deps = [
+        ":localization_to_region_calculator",
+        ":localization_to_region_calculator_cc_proto",
+        "//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:calculator_runner",
+        "//mediapipe/framework/formats:detection_cc_proto",
+        "//mediapipe/framework/formats:location_data_cc_proto",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/strings",
+    ],
+)

mediapipe/examples/desktop/autoflip/calculators/border_detection_calculator.cc

@@ -0,0 +1,302 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// This Calculator takes an ImageFrame and scales it appropriately.
+
+#include <algorithm>
+#include <memory>
+#include <vector>
+
+#include "mediapipe/examples/desktop/autoflip/autoflip_messages.pb.h"
+#include "mediapipe/examples/desktop/autoflip/calculators/border_detection_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/image_frame.h"
+#include "mediapipe/framework/formats/image_frame_opencv.h"
+#include "mediapipe/framework/port/opencv_core_inc.h"
+#include "mediapipe/framework/port/opencv_imgproc_inc.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+
+using mediapipe::Adopt;
+using mediapipe::CalculatorBase;
+using mediapipe::ImageFrame;
+using mediapipe::PacketTypeSet;
+using mediapipe::autoflip::Border;
+
+constexpr char kDetectedBorders[] = "DETECTED_BORDERS";
+constexpr int kMinBorderDistance = 5;
+constexpr int kKMeansClusterCount = 4;
+constexpr int kMaxPixelsToProcess = 300000;
+constexpr char kVideoInputTag[] = "VIDEO";
+
+namespace mediapipe {
+namespace autoflip {
+
+namespace {
+
+// Sets rect values into a proto.
+void SetRect(const cv::Rect& region,
+             const Border::RelativePosition& relative_position, Border* part) {
+  part->mutable_border_position()->set_x(region.x);
+  part->mutable_border_position()->set_y(region.y);
+  part->mutable_border_position()->set_width(region.width);
+  part->mutable_border_position()->set_height(region.height);
+  part->set_relative_position(relative_position);
+}
+
+}  // namespace
+
+// This calculator takes a sequence of images (video) and detects solid color
+// borders as well as the dominant color of the non-border area.  This per-frame
+// information is passed to downstream calculators.
+class BorderDetectionCalculator : public CalculatorBase {
+ public:
+  BorderDetectionCalculator() : frame_width_(-1), frame_height_(-1) {}
+  ~BorderDetectionCalculator() override {}
+  BorderDetectionCalculator(const BorderDetectionCalculator&) = delete;
+  BorderDetectionCalculator& operator=(const BorderDetectionCalculator&) =
+      delete;
+
+  static mediapipe::Status GetContract(mediapipe::CalculatorContract* cc);
+  mediapipe::Status Open(mediapipe::CalculatorContext* cc) override;
+  mediapipe::Status Process(mediapipe::CalculatorContext* cc) override;
+
+ private:
+  // Given a color and image direction, check to see if a border of that color
+  // exists.
+  void DetectBorder(const cv::Mat& frame, const Color& color,
+                    const Border::RelativePosition& direction,
+                    StaticFeatures* features);
+
+  // Provide the percent this color shows up in a given image.
+  double ColorCount(const Color& mask_color, const cv::Mat& image) const;
+
+  // Set member vars (image size) and confirm no changes frame-to-frame.
+  mediapipe::Status SetAndCheckInputs(const cv::Mat& frame);
+
+  // Find the dominant color for a input image.
+  double FindDominantColor(const cv::Mat& image, Color* dominant_color);
+
+  // Frame width and height.
+  int frame_width_;
+  int frame_height_;
+
+  // Options for processing.
+  BorderDetectionCalculatorOptions options_;
+};
+REGISTER_CALCULATOR(BorderDetectionCalculator);
+
+::mediapipe::Status BorderDetectionCalculator::Open(
+    mediapipe::CalculatorContext* cc) {
+  options_ = cc->Options<BorderDetectionCalculatorOptions>();
+  RET_CHECK_LT(options_.vertical_search_distance(), 0.5)
+      << "Search distance must be less than half the full image.";
+  return ::mediapipe::OkStatus();
+}
+
+mediapipe::Status BorderDetectionCalculator::SetAndCheckInputs(
+    const cv::Mat& frame) {
+  if (frame_width_ < 0) {
+    frame_width_ = frame.cols;
+  }
+  if (frame_height_ < 0) {
+    frame_height_ = frame.rows;
+  }
+  RET_CHECK_EQ(frame.cols, frame_width_)
+      << "Input frame dimensions must remain constant throughout the video.";
+  RET_CHECK_EQ(frame.rows, frame_height_)
+      << "Input frame dimensions must remain constant throughout the video.";
+  RET_CHECK_EQ(frame.channels(), 3) << "Input video type must be 3-channel";
+  return ::mediapipe::OkStatus();
+}
+
+mediapipe::Status BorderDetectionCalculator::Process(
+    mediapipe::CalculatorContext* cc) {
+  if (!cc->Inputs().HasTag(kVideoInputTag) ||
+      cc->Inputs().Tag(kVideoInputTag).Value().IsEmpty()) {
+    return ::mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
+           << "Input tag VIDEO not set or empty at timestamp: "
+           << cc->InputTimestamp().Value();
+  }
+  cv::Mat frame = mediapipe::formats::MatView(
+      &cc->Inputs().Tag(kVideoInputTag).Get<ImageFrame>());
+  MP_RETURN_IF_ERROR(SetAndCheckInputs(frame));
+
+  // Initialize output and set default values.
+  std::unique_ptr<StaticFeatures> features =
+      absl::make_unique<StaticFeatures>();
+  features->mutable_non_static_area()->set_x(0);
+  features->mutable_non_static_area()->set_width(frame_width_);
+  features->mutable_non_static_area()->set_y(options_.default_padding_px());
+  features->mutable_non_static_area()->set_height(
+      std::max(0, frame_height_ - options_.default_padding_px() * 2));
+
+  // Check for border at the top of the frame.
+  Color seed_color_top;
+  FindDominantColor(frame(cv::Rect(0, 0, frame_width_, 1)), &seed_color_top);
+  DetectBorder(frame, seed_color_top, Border::TOP, features.get());
+
+  // Check for border at the bottom of the frame.
+  Color seed_color_bottom;
+  FindDominantColor(frame(cv::Rect(0, frame_height_ - 1, frame_width_, 1)),
+                    &seed_color_bottom);
+  DetectBorder(frame, seed_color_bottom, Border::BOTTOM, features.get());
+
+  // Check the non-border area for a dominant color.
+  cv::Mat non_static_frame = frame(
+      cv::Rect(features->non_static_area().x(), features->non_static_area().y(),
+               features->non_static_area().width(),
+               features->non_static_area().height()));
+  Color dominant_color_nonborder;
+  double dominant_color_percent =
+      FindDominantColor(non_static_frame, &dominant_color_nonborder);
+  if (dominant_color_percent > options_.solid_background_tol_perc()) {
+    auto* bg_color = features->mutable_solid_background();
+    bg_color->set_r(dominant_color_nonborder.r());
+    bg_color->set_g(dominant_color_nonborder.g());
+    bg_color->set_b(dominant_color_nonborder.b());
+  }
+
+  // Output result.
+  cc->Outputs()
+      .Tag(kDetectedBorders)
+      .AddPacket(Adopt(features.release()).At(cc->InputTimestamp()));
+
+  return ::mediapipe::OkStatus();
+}
+
+//  Find the dominant color within an image.
+double BorderDetectionCalculator::FindDominantColor(const cv::Mat& image_raw,
+                                                    Color* dominant_color) {
+  cv::Mat image;
+  if (image_raw.total() > kMaxPixelsToProcess) {
+    float resize = kMaxPixelsToProcess / static_cast<float>(image_raw.total());
+    cv::resize(image_raw, image, cv::Size(), resize, resize);
+  } else {
+    image = image_raw;
+  }
+
+  cv::Mat float_data, cluster, cluster_center;
+  image.convertTo(float_data, CV_32F);
+  cv::Mat reshaped = float_data.reshape(1, float_data.total());
+
+  cv::kmeans(reshaped, kKMeansClusterCount, cluster,
+             cv::TermCriteria(CV_TERMCRIT_ITER, 5, 1.0), 1,
+             cv::KMEANS_PP_CENTERS, cluster_center);
+
+  std::vector<int> count(kKMeansClusterCount, 0);
+  for (int i = 0; i < cluster.rows; i++) {
+    count[cluster.at<int>(i, 0)]++;
+  }
+  auto max_cluster_ptr = std::max_element(count.begin(), count.end());
+  double max_cluster_perc =
+      *max_cluster_ptr / static_cast<double>(cluster.rows);
+  int max_cluster_idx = std::distance(count.begin(), max_cluster_ptr);
+
+  dominant_color->set_r(cluster_center.at<float>(max_cluster_idx, 2));
+  dominant_color->set_g(cluster_center.at<float>(max_cluster_idx, 1));
+  dominant_color->set_b(cluster_center.at<float>(max_cluster_idx, 0));
+
+  return max_cluster_perc;
+}
+
+double BorderDetectionCalculator::ColorCount(const Color& mask_color,
+                                             const cv::Mat& image) const {
+  int background_count = 0;
+  for (int i = 0; i < image.rows; i++) {
+    const uint8* row_ptr = image.ptr<uint8>(i);
+    for (int j = 0; j < image.cols * 3; j += 3) {
+      if (std::abs(mask_color.r() - static_cast<int>(row_ptr[j + 2])) <=
+              options_.color_tolerance() &&
+          std::abs(mask_color.g() - static_cast<int>(row_ptr[j + 1])) <=
+              options_.color_tolerance() &&
+          std::abs(mask_color.b() - static_cast<int>(row_ptr[j])) <=
+              options_.color_tolerance()) {
+        background_count++;
+      }
+    }
+  }
+  return background_count / static_cast<double>(image.rows * image.cols);
+}
+
+void BorderDetectionCalculator::DetectBorder(
+    const cv::Mat& frame, const Color& color,
+    const Border::RelativePosition& direction, StaticFeatures* features) {
+  // Search the entire image until we find an object, or hit the max search
+  // distance.
+  int search_distance =
+      (direction == Border::TOP || direction == Border::BOTTOM) ? frame.rows
+                                                                : frame.cols;
+  search_distance *= options_.vertical_search_distance();
+
+  // Check if each next line has a dominant color that matches the given
+  // border color.
+  int last_border = -1;
+  for (int i = 0; i < search_distance; i++) {
+    cv::Rect current_row;
+    switch (direction) {
+      case Border::TOP:
+        current_row = cv::Rect(0, i, frame.cols, 1);
+        break;
+      case Border::BOTTOM:
+        current_row = cv::Rect(0, frame.rows - i - 1, frame.cols, 1);
+        break;
+    }
+    if (ColorCount(color, frame(current_row)) <
+        options_.border_color_pixel_perc()) {
+      break;
+    }
+    last_border = i;
+  }
+
+  // Reject results that are not borders (or too small).
+  if (last_border <= kMinBorderDistance || last_border == search_distance - 1) {
+    return;
+  }
+
+  // Apply defined padding.
+  last_border += options_.border_object_padding_px();
+
+  switch (direction) {
+    case Border::TOP:
+      SetRect(cv::Rect(0, 0, frame.cols, last_border), Border::TOP,
+              features->add_border());
+      features->mutable_non_static_area()->set_y(
+          last_border + features->non_static_area().y());
+      features->mutable_non_static_area()->set_height(
+          std::max(0, frame_height_ - (features->non_static_area().y() +
+                                       options_.default_padding_px())));
+      break;
+    case Border::BOTTOM:
+      SetRect(
+          cv::Rect(0, frame.rows - last_border - 1, frame.cols, last_border),
+          Border::BOTTOM, features->add_border());
+
+      features->mutable_non_static_area()->set_height(std::max(
+          0, frame.rows - (features->non_static_area().y() + last_border +
+                           options_.default_padding_px())));
+
+      break;
+  }
+}
+
+::mediapipe::Status BorderDetectionCalculator::GetContract(
+    mediapipe::CalculatorContract* cc) {
+  cc->Inputs().Tag(kVideoInputTag).Set<ImageFrame>();
+  cc->Outputs().Tag(kDetectedBorders).Set<StaticFeatures>();
+  return ::mediapipe::OkStatus();
+}
+
+}  // namespace autoflip
+}  // namespace mediapipe

mediapipe/examples/desktop/autoflip/calculators/border_detection_calculator.proto

@@ -0,0 +1,44 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe.autoflip;
+
+import "mediapipe/framework/calculator.proto";
+
+// Next tag: 7
+message BorderDetectionCalculatorOptions {
+  extend mediapipe.CalculatorOptions {
+    optional BorderDetectionCalculatorOptions ext = 276599815;
+  }
+  // Max difference in color to be considered the same (per rgb channel).
+  optional int32 color_tolerance = 1 [default = 6];
+
+  // Amount of padding to add around any object within the border that is
+  // resized to fit into the new border.
+  optional int32 border_object_padding_px = 2 [default = 5];
+
+  // Distance (as a percent of height) to search for a border.
+  optional float vertical_search_distance = 3 [default = .20];
+
+  // Percent of pixels matching border color to be a border
+  optional float border_color_pixel_perc = 4 [default = .995];
+
+  // Percent of pixels matching background to be a solid background frame
+  optional float solid_background_tol_perc = 5 [default = .5];
+
+  // Force a border of this size in pixels on top and bottom.
+  optional int32 default_padding_px = 6 [default = 0];
+}

mediapipe/examples/desktop/autoflip/calculators/border_detection_calculator_test.cc

@@ -0,0 +1,397 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "absl/strings/string_view.h"
+#include "mediapipe/examples/desktop/autoflip/autoflip_messages.pb.h"
+#include "mediapipe/examples/desktop/autoflip/calculators/border_detection_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/formats/image_frame.h"
+#include "mediapipe/framework/formats/image_frame_opencv.h"
+#include "mediapipe/framework/port/benchmark.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/opencv_core_inc.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/port/status_matchers.h"
+
+using mediapipe::Adopt;
+using mediapipe::CalculatorGraphConfig;
+using mediapipe::CalculatorRunner;
+using mediapipe::ImageFormat;
+using mediapipe::ImageFrame;
+using mediapipe::Packet;
+using mediapipe::PacketTypeSet;
+using mediapipe::ParseTextProtoOrDie;
+using mediapipe::Timestamp;
+using mediapipe::autoflip::Border;
+
+namespace mediapipe {
+namespace autoflip {
+namespace {
+
+const char kConfig[] = R"(
+    calculator: "BorderDetectionCalculator"
+    input_stream: "VIDEO:camera_frames"
+    output_stream: "DETECTED_BORDERS:regions"
+    options:{
+    [mediapipe.autoflip.BorderDetectionCalculatorOptions.ext]:{
+      border_object_padding_px: 0
+    }
+    })";
+
+const char kConfigPad[] = R"(
+    calculator: "BorderDetectionCalculator"
+    input_stream: "VIDEO:camera_frames"
+    output_stream: "DETECTED_BORDERS:regions"
+    options:{
+    [mediapipe.autoflip.BorderDetectionCalculatorOptions.ext]:{
+      default_padding_px: 10
+      border_object_padding_px: 0
+    }
+    })";
+
+const int kTestFrameWidth = 640;
+const int kTestFrameHeight = 480;
+
+const int kTestFrameLargeWidth = 1920;
+const int kTestFrameLargeHeight = 1080;
+
+const int kTestFrameWidthTall = 1200;
+const int kTestFrameHeightTall = 2001;
+
+TEST(BorderDetectionCalculatorTest, NoBorderTest) {
+  auto runner = ::absl::make_unique<CalculatorRunner>(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfig));
+
+  auto input_frame = ::absl::make_unique<ImageFrame>(
+      ImageFormat::SRGB, kTestFrameWidth, kTestFrameHeight);
+  cv::Mat input_mat = mediapipe::formats::MatView(input_frame.get());
+  input_mat.setTo(cv::Scalar(0, 0, 0));
+  runner->MutableInputs()->Tag("VIDEO").packets.push_back(
+      Adopt(input_frame.release()).At(Timestamp::PostStream()));
+
+  // Run the calculator.
+  MP_ASSERT_OK(runner->Run());
+
+  const std::vector<Packet>& output_packets =
+      runner->Outputs().Tag("DETECTED_BORDERS").packets;
+  ASSERT_EQ(1, output_packets.size());
+  const auto& static_features = output_packets[0].Get<StaticFeatures>();
+  ASSERT_EQ(0, static_features.border().size());
+  EXPECT_EQ(0, static_features.non_static_area().x());
+  EXPECT_EQ(0, static_features.non_static_area().y());
+  EXPECT_EQ(kTestFrameWidth, static_features.non_static_area().width());
+  EXPECT_EQ(kTestFrameHeight, static_features.non_static_area().height());
+  EXPECT_TRUE(static_features.has_solid_background());
+  EXPECT_EQ(0, static_features.solid_background().r());
+  EXPECT_EQ(0, static_features.solid_background().g());
+  EXPECT_EQ(0, static_features.solid_background().b());
+}
+
+TEST(BorderDetectionCalculatorTest, TopBorderTest) {
+  auto runner = ::absl::make_unique<CalculatorRunner>(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfig));
+
+  const int kTopBorderHeight = 50;
+
+  auto input_frame = ::absl::make_unique<ImageFrame>(
+      ImageFormat::SRGB, kTestFrameWidth, kTestFrameHeight);
+  cv::Mat input_mat = mediapipe::formats::MatView(input_frame.get());
+  input_mat.setTo(cv::Scalar(0, 0, 0));
+  cv::Mat sub_image =
+      input_mat(cv::Rect(0, 0, kTestFrameWidth, kTopBorderHeight));
+  sub_image.setTo(cv::Scalar(255, 0, 0));
+  runner->MutableInputs()->Tag("VIDEO").packets.push_back(
+      Adopt(input_frame.release()).At(Timestamp::PostStream()));
+
+  // Run the calculator.
+  MP_ASSERT_OK(runner->Run());
+
+  const std::vector<Packet>& output_packets =
+      runner->Outputs().Tag("DETECTED_BORDERS").packets;
+  ASSERT_EQ(1, output_packets.size());
+  const auto& static_features = output_packets[0].Get<StaticFeatures>();
+  ASSERT_EQ(1, static_features.border().size());
+  const auto& part = static_features.border(0);
+  EXPECT_EQ(part.border_position().x(), 0);
+  EXPECT_EQ(part.border_position().y(), 0);
+  EXPECT_EQ(part.border_position().width(), kTestFrameWidth);
+  EXPECT_LT(std::abs(part.border_position().height() - kTopBorderHeight), 2);
+  EXPECT_TRUE(static_features.has_solid_background());
+  EXPECT_EQ(0, static_features.solid_background().r());
+  EXPECT_EQ(0, static_features.solid_background().g());
+  EXPECT_EQ(0, static_features.solid_background().b());
+  EXPECT_EQ(0, static_features.non_static_area().x());
+  EXPECT_EQ(kTopBorderHeight - 1, static_features.non_static_area().y());
+  EXPECT_EQ(kTestFrameWidth, static_features.non_static_area().width());
+  EXPECT_EQ(kTestFrameHeight - kTopBorderHeight + 1,
+            static_features.non_static_area().height());
+}
+
+TEST(BorderDetectionCalculatorTest, TopBorderPadTest) {
+  auto runner = ::absl::make_unique<CalculatorRunner>(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfigPad));
+
+  const int kTopBorderHeight = 50;
+
+  auto input_frame = ::absl::make_unique<ImageFrame>(
+      ImageFormat::SRGB, kTestFrameWidth, kTestFrameHeight);
+  cv::Mat input_mat = mediapipe::formats::MatView(input_frame.get());
+  input_mat.setTo(cv::Scalar(0, 0, 0));
+  cv::Mat sub_image =
+      input_mat(cv::Rect(0, 0, kTestFrameWidth, kTopBorderHeight));
+  sub_image.setTo(cv::Scalar(255, 0, 0));
+  runner->MutableInputs()->Tag("VIDEO").packets.push_back(
+      Adopt(input_frame.release()).At(Timestamp::PostStream()));
+
+  // Run the calculator.
+  MP_ASSERT_OK(runner->Run());
+
+  const std::vector<Packet>& output_packets =
+      runner->Outputs().Tag("DETECTED_BORDERS").packets;
+  ASSERT_EQ(1, output_packets.size());
+  const auto& static_features = output_packets[0].Get<StaticFeatures>();
+  ASSERT_EQ(1, static_features.border().size());
+  const auto& part = static_features.border(0);
+  EXPECT_EQ(part.border_position().x(), 0);
+  EXPECT_EQ(part.border_position().y(), 0);
+  EXPECT_EQ(part.border_position().width(), kTestFrameWidth);
+  EXPECT_LT(std::abs(part.border_position().height() - kTopBorderHeight), 2);
+  EXPECT_TRUE(static_features.has_solid_background());
+  EXPECT_EQ(0, static_features.solid_background().r());
+  EXPECT_EQ(0, static_features.solid_background().g());
+  EXPECT_EQ(0, static_features.solid_background().b());
+  EXPECT_EQ(Border::TOP, part.relative_position());
+  EXPECT_EQ(0, static_features.non_static_area().x());
+  EXPECT_EQ(9 + kTopBorderHeight, static_features.non_static_area().y());
+  EXPECT_EQ(kTestFrameWidth, static_features.non_static_area().width());
+  EXPECT_EQ(kTestFrameHeight - 19 - kTopBorderHeight,
+            static_features.non_static_area().height());
+}
+
+TEST(BorderDetectionCalculatorTest, BottomBorderTest) {
+  auto runner = ::absl::make_unique<CalculatorRunner>(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfig));
+
+  const int kBottomBorderHeight = 50;
+
+  auto input_frame = ::absl::make_unique<ImageFrame>(
+      ImageFormat::SRGB, kTestFrameWidth, kTestFrameHeight);
+  cv::Mat input_mat = mediapipe::formats::MatView(input_frame.get());
+  input_mat.setTo(cv::Scalar(0, 0, 0));
+  cv::Mat bottom_image =
+      input_mat(cv::Rect(0, kTestFrameHeight - kBottomBorderHeight,
+                         kTestFrameWidth, kBottomBorderHeight));
+  bottom_image.setTo(cv::Scalar(255, 0, 0));
+  runner->MutableInputs()->Tag("VIDEO").packets.push_back(
+      Adopt(input_frame.release()).At(Timestamp::PostStream()));
+
+  // Run the calculator.
+  MP_ASSERT_OK(runner->Run());
+
+  const std::vector<Packet>& output_packets =
+      runner->Outputs().Tag("DETECTED_BORDERS").packets;
+  ASSERT_EQ(1, output_packets.size());
+  const auto& static_features = output_packets[0].Get<StaticFeatures>();
+  ASSERT_EQ(1, static_features.border().size());
+  const auto& part = static_features.border(0);
+  EXPECT_EQ(part.border_position().x(), 0);
+  EXPECT_EQ(part.border_position().y(), kTestFrameHeight - kBottomBorderHeight);
+  EXPECT_EQ(part.border_position().width(), kTestFrameWidth);
+  EXPECT_LT(std::abs(part.border_position().height() - kBottomBorderHeight), 2);
+  EXPECT_TRUE(static_features.has_solid_background());
+  EXPECT_EQ(0, static_features.solid_background().r());
+  EXPECT_EQ(0, static_features.solid_background().g());
+  EXPECT_EQ(0, static_features.solid_background().b());
+  EXPECT_EQ(Border::BOTTOM, part.relative_position());
+}
+
+TEST(BorderDetectionCalculatorTest, TopBottomBorderTest) {
+  auto runner = ::absl::make_unique<CalculatorRunner>(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfig));
+
+  const int kBottomBorderHeight = 50;
+  const int kTopBorderHeight = 25;
+
+  auto input_frame = ::absl::make_unique<ImageFrame>(
+      ImageFormat::SRGB, kTestFrameWidth, kTestFrameHeight);
+  cv::Mat input_mat = mediapipe::formats::MatView(input_frame.get());
+  input_mat.setTo(cv::Scalar(0, 0, 0));
+  cv::Mat top_image =
+      input_mat(cv::Rect(0, 0, kTestFrameWidth, kTopBorderHeight));
+  top_image.setTo(cv::Scalar(0, 255, 0));
+  cv::Mat bottom_image =
+      input_mat(cv::Rect(0, kTestFrameHeight - kBottomBorderHeight,
+                         kTestFrameWidth, kBottomBorderHeight));
+  bottom_image.setTo(cv::Scalar(255, 0, 0));
+  runner->MutableInputs()->Tag("VIDEO").packets.push_back(
+      Adopt(input_frame.release()).At(Timestamp::PostStream()));
+
+  // Run the calculator.
+  MP_ASSERT_OK(runner->Run());
+
+  const std::vector<Packet>& output_packets =
+      runner->Outputs().Tag("DETECTED_BORDERS").packets;
+  ASSERT_EQ(1, output_packets.size());
+  const auto& static_features = output_packets[0].Get<StaticFeatures>();
+  ASSERT_EQ(2, static_features.border().size());
+  auto part = static_features.border(0);
+  EXPECT_EQ(part.border_position().x(), 0);
+  EXPECT_EQ(part.border_position().y(), 0);
+  EXPECT_EQ(part.border_position().width(), kTestFrameWidth);
+  EXPECT_LT(std::abs(part.border_position().height() - kTopBorderHeight), 2);
+  EXPECT_TRUE(static_features.has_solid_background());
+  EXPECT_EQ(0, static_features.solid_background().r());
+  EXPECT_EQ(0, static_features.solid_background().g());
+  EXPECT_EQ(0, static_features.solid_background().b());
+  EXPECT_EQ(0, static_features.non_static_area().x());
+  EXPECT_EQ(kTopBorderHeight - 1, static_features.non_static_area().y());
+  EXPECT_EQ(kTestFrameWidth, static_features.non_static_area().width());
+  EXPECT_EQ(kTestFrameHeight - kTopBorderHeight - kBottomBorderHeight + 2,
+            static_features.non_static_area().height());
+  EXPECT_EQ(Border::TOP, part.relative_position());
+
+  part = static_features.border(1);
+  EXPECT_EQ(part.border_position().x(), 0);
+  EXPECT_EQ(part.border_position().y(), kTestFrameHeight - kBottomBorderHeight);
+  EXPECT_EQ(part.border_position().width(), kTestFrameWidth);
+  EXPECT_LT(std::abs(part.border_position().height() - kBottomBorderHeight), 2);
+  EXPECT_EQ(Border::BOTTOM, part.relative_position());
+}
+
+TEST(BorderDetectionCalculatorTest, TopBottomBorderTestAspect2) {
+  auto runner = ::absl::make_unique<CalculatorRunner>(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfig));
+
+  const int kBottomBorderHeight = 50;
+  const int kTopBorderHeight = 25;
+
+  auto input_frame = ::absl::make_unique<ImageFrame>(
+      ImageFormat::SRGB, kTestFrameWidthTall, kTestFrameHeightTall);
+  cv::Mat input_mat = mediapipe::formats::MatView(input_frame.get());
+  input_mat.setTo(cv::Scalar(0, 0, 0));
+  cv::Mat top_image =
+      input_mat(cv::Rect(0, 0, kTestFrameWidthTall, kTopBorderHeight));
+  top_image.setTo(cv::Scalar(0, 255, 0));
+  cv::Mat bottom_image =
+      input_mat(cv::Rect(0, kTestFrameHeightTall - kBottomBorderHeight,
+                         kTestFrameWidthTall, kBottomBorderHeight));
+  bottom_image.setTo(cv::Scalar(255, 0, 0));
+  runner->MutableInputs()->Tag("VIDEO").packets.push_back(
+      Adopt(input_frame.release()).At(Timestamp::PostStream()));
+
+  // Run the calculator.
+  MP_ASSERT_OK(runner->Run());
+
+  const std::vector<Packet>& output_packets =
+      runner->Outputs().Tag("DETECTED_BORDERS").packets;
+  ASSERT_EQ(1, output_packets.size());
+  const auto& static_features = output_packets[0].Get<StaticFeatures>();
+  ASSERT_EQ(2, static_features.border().size());
+  auto part = static_features.border(0);
+  EXPECT_EQ(part.border_position().x(), 0);
+  EXPECT_EQ(part.border_position().y(), 0);
+  EXPECT_EQ(part.border_position().width(), kTestFrameWidthTall);
+  EXPECT_LT(std::abs(part.border_position().height() - kTopBorderHeight), 2);
+  EXPECT_TRUE(static_features.has_solid_background());
+  EXPECT_EQ(0, static_features.solid_background().r());
+  EXPECT_EQ(0, static_features.solid_background().g());
+  EXPECT_EQ(0, static_features.solid_background().b());
+  EXPECT_EQ(Border::TOP, part.relative_position());
+
+  part = static_features.border(1);
+  EXPECT_EQ(part.border_position().x(), 0);
+  EXPECT_EQ(part.border_position().y(),
+            kTestFrameHeightTall - kBottomBorderHeight);
+  EXPECT_EQ(part.border_position().width(), kTestFrameWidthTall);
+  EXPECT_LT(std::abs(part.border_position().height() - kBottomBorderHeight), 2);
+  EXPECT_TRUE(static_features.has_solid_background());
+  EXPECT_EQ(0, static_features.solid_background().r());
+  EXPECT_EQ(0, static_features.solid_background().g());
+  EXPECT_EQ(0, static_features.solid_background().b());
+  EXPECT_EQ(Border::BOTTOM, part.relative_position());
+}
+
+TEST(BorderDetectionCalculatorTest, DominantColor) {
+  CalculatorGraphConfig::Node node =
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfigPad);
+  node.mutable_options()
+      ->MutableExtension(BorderDetectionCalculatorOptions::ext)
+      ->set_solid_background_tol_perc(.25);
+
+  auto runner = ::absl::make_unique<CalculatorRunner>(node);
+
+  auto input_frame = ::absl::make_unique<ImageFrame>(
+      ImageFormat::SRGB, kTestFrameWidth, kTestFrameHeight);
+  cv::Mat input_mat = mediapipe::formats::MatView(input_frame.get());
+  input_mat.setTo(cv::Scalar(0, 0, 0));
+
+  cv::Mat sub_image = input_mat(cv::Rect(
+      kTestFrameWidth / 2, 0, kTestFrameWidth / 2, kTestFrameHeight / 2));
+  sub_image.setTo(cv::Scalar(0, 255, 0));
+
+  sub_image = input_mat(cv::Rect(0, kTestFrameHeight / 2, kTestFrameWidth / 2,
+                                 kTestFrameHeight / 2));
+  sub_image.setTo(cv::Scalar(0, 0, 255));
+
+  sub_image =
+      input_mat(cv::Rect(0, 0, kTestFrameWidth / 2 + 50, kTestFrameHeight / 2));
+  sub_image.setTo(cv::Scalar(255, 0, 0));
+
+  runner->MutableInputs()->Tag("VIDEO").packets.push_back(
+      Adopt(input_frame.release()).At(Timestamp::PostStream()));
+
+  // Run the calculator.
+  MP_ASSERT_OK(runner->Run());
+
+  const std::vector<Packet>& output_packets =
+      runner->Outputs().Tag("DETECTED_BORDERS").packets;
+  ASSERT_EQ(1, output_packets.size());
+  const auto& static_features = output_packets[0].Get<StaticFeatures>();
+  ASSERT_EQ(0, static_features.border().size());
+  ASSERT_TRUE(static_features.has_solid_background());
+  EXPECT_EQ(0, static_features.solid_background().r());
+  EXPECT_EQ(0, static_features.solid_background().g());
+  EXPECT_EQ(255, static_features.solid_background().b());
+}
+
+void BM_Large(benchmark::State& state) {
+  for (auto _ : state) {
+    auto runner = ::absl::make_unique<CalculatorRunner>(
+        ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfig));
+
+    const int kTopBorderHeight = 50;
+
+    auto input_frame = ::absl::make_unique<ImageFrame>(
+        ImageFormat::SRGB, kTestFrameLargeWidth, kTestFrameLargeHeight);
+    cv::Mat input_mat = mediapipe::formats::MatView(input_frame.get());
+    input_mat.setTo(cv::Scalar(0, 0, 0));
+    cv::Mat sub_image =
+        input_mat(cv::Rect(0, 0, kTestFrameLargeWidth, kTopBorderHeight));
+    sub_image.setTo(cv::Scalar(255, 0, 0));
+    runner->MutableInputs()->Tag("VIDEO").packets.push_back(
+        Adopt(input_frame.release()).At(Timestamp::PostStream()));
+
+    // Run the calculator.
+    MP_ASSERT_OK(runner->Run());
+  }
+}
+BENCHMARK(BM_Large);
+
+}  // namespace
+}  // namespace autoflip
+}  // namespace mediapipe

mediapipe/examples/desktop/autoflip/calculators/face_to_region_calculator.cc

@@ -0,0 +1,269 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <algorithm>
+#include <memory>
+
+#include "mediapipe/examples/desktop/autoflip/autoflip_messages.pb.h"
+#include "mediapipe/examples/desktop/autoflip/calculators/face_to_region_calculator.pb.h"
+#include "mediapipe/examples/desktop/autoflip/quality/visual_scorer.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/detection.pb.h"
+#include "mediapipe/framework/formats/image_frame.h"
+#include "mediapipe/framework/formats/image_frame_opencv.h"
+#include "mediapipe/framework/formats/location_data.pb.h"
+#include "mediapipe/framework/port/opencv_core_inc.h"
+#include "mediapipe/framework/port/opencv_imgproc_inc.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/port/status_builder.h"
+
+namespace mediapipe {
+namespace autoflip {
+
+// This calculator converts detected faces to SalientRegion protos that can be
+// used for downstream processing. Each SalientRegion is scored using image
+// cues. Scoring can be controlled through
+// FaceToRegionCalculator::scorer_options.
+// Example:
+//    calculator: "FaceToRegionCalculator"
+//    input_stream: "VIDEO:frames"
+//    input_stream: "FACES:faces"
+//    output_stream: "REGIONS:regions"
+//    options:{
+//      [mediapipe.autoflip.FaceToRegionCalculatorOptions.ext]:{
+//        export_individual_face_landmarks: false
+//        export_whole_face: true
+//      }
+//    }
+//
+class FaceToRegionCalculator : public CalculatorBase {
+ public:
+  FaceToRegionCalculator();
+  ~FaceToRegionCalculator() override {}
+  FaceToRegionCalculator(const FaceToRegionCalculator&) = delete;
+  FaceToRegionCalculator& operator=(const FaceToRegionCalculator&) = delete;
+
+  static ::mediapipe::Status GetContract(mediapipe::CalculatorContract* cc);
+  ::mediapipe::Status Open(mediapipe::CalculatorContext* cc) override;
+  ::mediapipe::Status Process(mediapipe::CalculatorContext* cc) override;
+
+ private:
+  double NormalizeX(const int pixel);
+  double NormalizeY(const int pixel);
+  // Extend the given SalientRegion to include the given point.
+  void ExtendSalientRegionWithPoint(const float x, const float y,
+                                    SalientRegion* region);
+  // Calculator options.
+  FaceToRegionCalculatorOptions options_;
+
+  // A scorer used to assign weights to faces.
+  std::unique_ptr<VisualScorer> scorer_;
+  // Dimensions of video frame
+  int frame_width_;
+  int frame_height_;
+};
+REGISTER_CALCULATOR(FaceToRegionCalculator);
+
+FaceToRegionCalculator::FaceToRegionCalculator() {}
+
+::mediapipe::Status FaceToRegionCalculator::GetContract(
+    mediapipe::CalculatorContract* cc) {
+  cc->Inputs().Tag("VIDEO").Set<ImageFrame>();
+  cc->Inputs().Tag("FACES").Set<std::vector<mediapipe::Detection>>();
+  cc->Outputs().Tag("REGIONS").Set<DetectionSet>();
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status FaceToRegionCalculator::Open(
+    mediapipe::CalculatorContext* cc) {
+  options_ = cc->Options<FaceToRegionCalculatorOptions>();
+  scorer_ = absl::make_unique<VisualScorer>(options_.scorer_options());
+  frame_width_ = -1;
+  frame_height_ = -1;
+  return ::mediapipe::OkStatus();
+}
+
+inline double FaceToRegionCalculator::NormalizeX(const int pixel) {
+  return pixel / static_cast<double>(frame_width_);
+}
+
+inline double FaceToRegionCalculator::NormalizeY(const int pixel) {
+  return pixel / static_cast<double>(frame_height_);
+}
+
+void FaceToRegionCalculator::ExtendSalientRegionWithPoint(
+    const float x, const float y, SalientRegion* region) {
+  auto* location = region->mutable_location_normalized();
+  if (!location->has_width()) {
+    location->set_width(NormalizeX(1));
+  } else if (x < location->x()) {
+    location->set_width(location->width() + location->x() - x);
+  } else if (x > location->x() + location->width()) {
+    location->set_width(x - location->x());
+  }
+  if (!location->has_height()) {
+    location->set_height(NormalizeY(1));
+  } else if (y < location->y()) {
+    location->set_height(location->height() + location->y() - y);
+  } else if (y > location->y() + location->height()) {
+    location->set_height(y - location->y());
+  }
+
+  if (!location->has_x()) {
+    location->set_x(x);
+  } else {
+    location->set_x(std::min(location->x(), x));
+  }
+  if (!location->has_y()) {
+    location->set_y(y);
+  } else {
+    location->set_y(std::min(location->y(), y));
+  }
+}
+
+::mediapipe::Status FaceToRegionCalculator::Process(
+    mediapipe::CalculatorContext* cc) {
+  if (cc->Inputs().Tag("VIDEO").Value().IsEmpty()) {
+    return ::mediapipe::UnknownErrorBuilder(MEDIAPIPE_LOC) << "No VIDEO input.";
+  }
+
+  cv::Mat frame =
+      mediapipe::formats::MatView(&cc->Inputs().Tag("VIDEO").Get<ImageFrame>());
+  frame_width_ = frame.cols;
+  frame_height_ = frame.rows;
+
+  auto region_set = ::absl::make_unique<DetectionSet>();
+  if (!cc->Inputs().Tag("FACES").Value().IsEmpty()) {
+    const auto& input_faces =
+        cc->Inputs().Tag("FACES").Get<std::vector<mediapipe::Detection>>();
+
+    for (const auto& input_face : input_faces) {
+      RET_CHECK(input_face.location_data().format() ==
+                mediapipe::LocationData::RELATIVE_BOUNDING_BOX)
+          << "Face detection input is lacking required relative_bounding_box()";
+      // 6 landmarks should be provided, ordered as:
+      // Left eye, Right eye, Nose tip, Mouth center, Left ear tragion, Right
+      // ear tragion.
+      RET_CHECK(input_face.location_data().relative_keypoints().size() == 6)
+          << "Face detection input expected 6 keypoints, has "
+          << input_face.location_data().relative_keypoints().size();
+
+      const auto& location = input_face.location_data().relative_bounding_box();
+
+      // Reduce region size to only contain parts of the image in frame.
+      float x = std::max(0.0f, location.xmin());
+      float y = std::max(0.0f, location.ymin());
+      float width =
+          std::min(location.width() - abs(x - location.xmin()), 1 - x);
+      float height =
+          std::min(location.height() - abs(y - location.ymin()), 1 - y);
+
+      // Convert the face to a region.
+      if (options_.export_whole_face()) {
+        SalientRegion* region = region_set->add_detections();
+        region->mutable_location_normalized()->set_x(x);
+        region->mutable_location_normalized()->set_y(y);
+        region->mutable_location_normalized()->set_width(width);
+        region->mutable_location_normalized()->set_height(height);
+        region->mutable_signal_type()->set_standard(SignalType::FACE_FULL);
+
+        // Score the face based on image cues.
+        float visual_score = 1.0f;
+        if (options_.use_visual_scorer()) {
+          MP_RETURN_IF_ERROR(
+              scorer_->CalculateScore(frame, *region, &visual_score));
+        }
+        region->set_score(visual_score);
+      }
+
+      // Generate two more output regions from important face landmarks. One
+      // includes all exterior landmarks, such as ears and chin, and the
+      // other includes only interior landmarks, such as the eye edges and the
+      // mouth.
+      SalientRegion core_landmark_region, all_landmark_region;
+      // Keypoints are ordered: Left Eye, Right Eye, Nose Tip, Mouth Center,
+      // Left Ear Tragion, Right Ear Tragion.
+
+      // Set 'core' landmarks (Left Eye, Right Eye, Nose Tip, Mouth Center)
+      for (int i = 0; i < 4; i++) {
+        const auto& keypoint = input_face.location_data().relative_keypoints(i);
+        if (options_.export_individual_face_landmarks()) {
+          SalientRegion* region = region_set->add_detections();
+          region->mutable_location_normalized()->set_x(keypoint.x());
+          region->mutable_location_normalized()->set_y(keypoint.y());
+          region->mutable_location_normalized()->set_width(NormalizeX(1));
+          region->mutable_location_normalized()->set_height(NormalizeY(1));
+          region->mutable_signal_type()->set_standard(
+              SignalType::FACE_LANDMARK);
+        }
+
+        // Extend the core/full landmark regions to include the new
+        ExtendSalientRegionWithPoint(keypoint.x(), keypoint.y(),
+                                     &core_landmark_region);
+        ExtendSalientRegionWithPoint(keypoint.x(), keypoint.y(),
+                                     &all_landmark_region);
+      }
+      // Set 'all' landmarks (Left Ear Tragion, Right Ear Tragion + core)
+      for (int i = 4; i < 6; i++) {
+        const auto& keypoint = input_face.location_data().relative_keypoints(i);
+        if (options_.export_individual_face_landmarks()) {
+          SalientRegion* region = region_set->add_detections();
+          region->mutable_location()->set_x(keypoint.x());
+          region->mutable_location()->set_y(keypoint.y());
+          region->mutable_location()->set_width(NormalizeX(1));
+          region->mutable_location()->set_height(NormalizeY(1));
+          region->mutable_signal_type()->set_standard(
+              SignalType::FACE_LANDMARK);
+        }
+
+        // Extend the full landmark region to include the new landmark.
+        ExtendSalientRegionWithPoint(keypoint.x(), keypoint.y(),
+                                     &all_landmark_region);
+      }
+
+      // Generate scores for the landmark bboxes and export them.
+      if (options_.export_bbox_from_landmarks() &&
+          core_landmark_region.has_location_normalized()) {  // Not empty.
+        float visual_score = 1.0f;
+        if (options_.use_visual_scorer()) {
+          MP_RETURN_IF_ERROR(scorer_->CalculateScore(
+              frame, core_landmark_region, &visual_score));
+        }
+        core_landmark_region.set_score(visual_score);
+        core_landmark_region.mutable_signal_type()->set_standard(
+            SignalType::FACE_CORE_LANDMARKS);
+        *region_set->add_detections() = core_landmark_region;
+      }
+      if (options_.export_bbox_from_landmarks() &&
+          all_landmark_region.has_location_normalized()) {  // Not empty.
+        float visual_score = 1.0f;
+        if (options_.use_visual_scorer()) {
+          MP_RETURN_IF_ERROR(scorer_->CalculateScore(frame, all_landmark_region,
+                                                     &visual_score));
+        }
+        all_landmark_region.set_score(visual_score);
+        all_landmark_region.mutable_signal_type()->set_standard(
+            SignalType::FACE_ALL_LANDMARKS);
+        *region_set->add_detections() = all_landmark_region;
+      }
+    }
+  }
+  cc->Outputs().Tag("REGIONS").Add(region_set.release(), cc->InputTimestamp());
+
+  return ::mediapipe::OkStatus();
+}
+
+}  // namespace autoflip
+}  // namespace mediapipe

mediapipe/examples/desktop/autoflip/calculators/face_to_region_calculator.proto

@@ -0,0 +1,50 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe.autoflip;
+
+import "mediapipe/examples/desktop/autoflip/quality/visual_scorer.proto";
+import "mediapipe/framework/calculator.proto";
+
+// Next tag: 6
+message FaceToRegionCalculatorOptions {
+  extend mediapipe.CalculatorOptions {
+    optional FaceToRegionCalculatorOptions ext = 282401234;
+  }
+
+  // Options for generating a score for the entire face from its visual
+  // appearance. The generated score is used to modulate the detection scores
+  // for whole face and/or landmark bbox region types.
+  optional VisualScorerOptions scorer_options = 1;
+
+  // If true, export the large face bounding box generated by the face tracker.
+  // This bounding box is generally larger than the actual face and relatively
+  // inaccurate.
+  optional bool export_whole_face = 2 [default = false];
+
+  // If true, export a number of individual face landmarks (eyes, nose, mouth,
+  // ears etc) as separate SalientRegion protos.
+  optional bool export_individual_face_landmarks = 3 [default = false];
+
+  // If true, export two bounding boxes from landmarks (one for the core face
+  // landmarks like eyes and nose, and one for extended landmarks including ears
+  // and chin).
+  optional bool export_bbox_from_landmarks = 4 [default = true];
+
+  // If true, generate a score from the appearance of the face and use it to
+  // modulate the detection scores for whole face and/or landmark bboxes.
+  optional bool use_visual_scorer = 5 [default = true];
+}

mediapipe/examples/desktop/autoflip/calculators/face_to_region_calculator_test.cc

@@ -0,0 +1,247 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "absl/strings/string_view.h"
+#include "mediapipe/examples/desktop/autoflip/autoflip_messages.pb.h"
+#include "mediapipe/examples/desktop/autoflip/calculators/face_to_region_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/formats/detection.pb.h"
+#include "mediapipe/framework/formats/image_frame.h"
+#include "mediapipe/framework/formats/image_frame_opencv.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/port/status_matchers.h"
+
+using mediapipe::Detection;
+
+namespace mediapipe {
+namespace autoflip {
+namespace {
+
+const char kConfig[] = R"(
+    calculator: "FaceToRegionCalculator"
+    input_stream: "VIDEO:frames"
+    input_stream: "FACES:faces"
+    output_stream: "REGIONS:regions"
+    )";
+
+const char kFace1[] = R"(location_data {
+           format: RELATIVE_BOUNDING_BOX
+           relative_bounding_box {
+             xmin: -0.00375
+             ymin: 0.003333
+             width: 0.125
+             height: 0.33333
+           }
+           relative_keypoints { x: 0.03125 y: 0.05 }
+           relative_keypoints { x: 0.0875 y: 0.0666666 }
+           relative_keypoints { x: 0.03125 y: 0.05 }
+           relative_keypoints { x: 0.0875 y: 0.0666666 }
+           relative_keypoints { x: 0.0250 y: 0.0666666 }
+           relative_keypoints { x: 0.0950 y: 0.0666666 }
+         })";
+
+const char kFace2[] = R"(location_data {
+           format: RELATIVE_BOUNDING_BOX
+           relative_bounding_box {
+             xmin: 0.0025
+             ymin: 0.005
+             width: 0.25
+             height: 0.5
+           }
+           relative_keypoints { x: 0 y: 0 }
+           relative_keypoints { x: 0 y: 0 }
+           relative_keypoints { x: 0 y: 0 }
+           relative_keypoints { x: 0 y: 0 }
+           relative_keypoints { x: 0 y: 0 }
+           relative_keypoints { x: 0 y: 0 }
+         })";
+
+const char kFace3[] = R"(location_data {
+           format: RELATIVE_BOUNDING_BOX
+           relative_bounding_box {
+             xmin: 0.0
+             ymin: 0.0
+             width: 0.5
+             height: 0.5
+           }
+           relative_keypoints { x: 0 y: 0 }
+           relative_keypoints { x: 0 y: 0 }
+           relative_keypoints { x: 0 y: 0 }
+           relative_keypoints { x: 0 y: 0 }
+           relative_keypoints { x: 0 y: 0 }
+           relative_keypoints { x: 0 y: 0 }
+         })";
+
+void SetInputs(CalculatorRunner* runner,
+               const std::vector<std::string>& faces) {
+  // Setup an input video frame.
+  auto input_frame =
+      ::absl::make_unique<ImageFrame>(ImageFormat::SRGB, 800, 600);
+  runner->MutableInputs()->Tag("VIDEO").packets.push_back(
+      Adopt(input_frame.release()).At(Timestamp::PostStream()));
+  // Setup two faces as input.
+  auto input_faces = ::absl::make_unique<std::vector<Detection>>();
+  // A face with landmarks.
+  for (const auto& face : faces) {
+    input_faces->push_back(ParseTextProtoOrDie<Detection>(face));
+  }
+  runner->MutableInputs()->Tag("FACES").packets.push_back(
+      Adopt(input_faces.release()).At(Timestamp::PostStream()));
+}
+
+CalculatorGraphConfig::Node MakeConfig(bool whole_face, bool landmarks,
+                                       bool bb_from_landmarks) {
+  auto config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfig);
+
+  config.mutable_options()
+      ->MutableExtension(FaceToRegionCalculatorOptions::ext)
+      ->set_export_whole_face(whole_face);
+
+  config.mutable_options()
+      ->MutableExtension(FaceToRegionCalculatorOptions::ext)
+      ->set_export_individual_face_landmarks(landmarks);
+
+  config.mutable_options()
+      ->MutableExtension(FaceToRegionCalculatorOptions::ext)
+      ->set_export_bbox_from_landmarks(bb_from_landmarks);
+
+  return config;
+}
+
+TEST(FaceToRegionCalculatorTest, FaceFullTypeSize) {
+  // Setup test
+  auto runner =
+      ::absl::make_unique<CalculatorRunner>(MakeConfig(true, false, false));
+  SetInputs(runner.get(), {kFace1, kFace2});
+
+  // Run the calculator.
+  MP_ASSERT_OK(runner->Run());
+
+  // Check the output regions.
+  const std::vector<Packet>& output_packets =
+      runner->Outputs().Tag("REGIONS").packets;
+  ASSERT_EQ(1, output_packets.size());
+
+  const auto& regions = output_packets[0].Get<DetectionSet>();
+  ASSERT_EQ(2, regions.detections().size());
+  auto face_1 = regions.detections(0);
+  EXPECT_EQ(face_1.signal_type().standard(), SignalType::FACE_FULL);
+  EXPECT_FLOAT_EQ(face_1.location_normalized().x(), 0);
+  EXPECT_FLOAT_EQ(face_1.location_normalized().y(), 0.003333);
+  EXPECT_FLOAT_EQ(face_1.location_normalized().width(), 0.12125);
+  EXPECT_FLOAT_EQ(face_1.location_normalized().height(), 0.33333);
+  EXPECT_FLOAT_EQ(face_1.score(), 0.040214583);
+
+  auto face_2 = regions.detections(1);
+  EXPECT_EQ(face_2.signal_type().standard(), SignalType::FACE_FULL);
+  EXPECT_FLOAT_EQ(face_2.location_normalized().x(), 0.0025);
+  EXPECT_FLOAT_EQ(face_2.location_normalized().y(), 0.005);
+  EXPECT_FLOAT_EQ(face_2.location_normalized().width(), 0.25);
+  EXPECT_FLOAT_EQ(face_2.location_normalized().height(), 0.5);
+  EXPECT_FLOAT_EQ(face_2.score(), 0.125);
+}
+
+TEST(FaceToRegionCalculatorTest, FaceLandmarksTypeSize) {
+  // Setup test
+  auto runner =
+      ::absl::make_unique<CalculatorRunner>(MakeConfig(false, true, false));
+  SetInputs(runner.get(), {kFace1});
+
+  // Run the calculator.
+  MP_ASSERT_OK(runner->Run());
+
+  // Check the output regions.
+  const std::vector<Packet>& output_packets =
+      runner->Outputs().Tag("REGIONS").packets;
+  ASSERT_EQ(1, output_packets.size());
+
+  const auto& regions = output_packets[0].Get<DetectionSet>();
+  ASSERT_EQ(6, regions.detections().size());
+  auto landmark_1 = regions.detections(0);
+  EXPECT_EQ(landmark_1.signal_type().standard(), SignalType::FACE_LANDMARK);
+  EXPECT_FLOAT_EQ(landmark_1.location_normalized().x(), 0.03125);
+  EXPECT_FLOAT_EQ(landmark_1.location_normalized().y(), 0.05);
+  EXPECT_FLOAT_EQ(landmark_1.location_normalized().width(), 0.00125);
+  EXPECT_FLOAT_EQ(landmark_1.location_normalized().height(), 0.0016666667);
+
+  auto landmark_2 = regions.detections(1);
+  EXPECT_EQ(landmark_2.signal_type().standard(), SignalType::FACE_LANDMARK);
+  EXPECT_FLOAT_EQ(landmark_2.location_normalized().x(), 0.0875);
+  EXPECT_FLOAT_EQ(landmark_2.location_normalized().y(), 0.0666666);
+  EXPECT_FLOAT_EQ(landmark_2.location_normalized().width(), 0.00125);
+  EXPECT_FLOAT_EQ(landmark_2.location_normalized().height(), 0.0016666667);
+}
+
+TEST(FaceToRegionCalculatorTest, FaceLandmarksBox) {
+  // Setup test
+  auto runner =
+      ::absl::make_unique<CalculatorRunner>(MakeConfig(false, false, true));
+  SetInputs(runner.get(), {kFace1});
+
+  // Run the calculator.
+  MP_ASSERT_OK(runner->Run());
+
+  // Check the output regions.
+  const std::vector<Packet>& output_packets =
+      runner->Outputs().Tag("REGIONS").packets;
+  ASSERT_EQ(1, output_packets.size());
+
+  const auto& regions = output_packets[0].Get<DetectionSet>();
+  ASSERT_EQ(2, regions.detections().size());
+  auto landmark_1 = regions.detections(0);
+  EXPECT_EQ(landmark_1.signal_type().standard(),
+            SignalType::FACE_CORE_LANDMARKS);
+  EXPECT_FLOAT_EQ(landmark_1.location_normalized().x(), 0.03125);
+  EXPECT_FLOAT_EQ(landmark_1.location_normalized().y(), 0.05);
+  EXPECT_FLOAT_EQ(landmark_1.location_normalized().width(), 0.056249999);
+  EXPECT_FLOAT_EQ(landmark_1.location_normalized().height(), 0.016666602);
+  EXPECT_FLOAT_EQ(landmark_1.score(), 0.00084375002);
+
+  auto landmark_2 = regions.detections(1);
+  EXPECT_EQ(landmark_2.signal_type().standard(),
+            SignalType::FACE_ALL_LANDMARKS);
+  EXPECT_FLOAT_EQ(landmark_2.location_normalized().x(), 0.025);
+  EXPECT_FLOAT_EQ(landmark_2.location_normalized().y(), 0.050000001);
+  EXPECT_FLOAT_EQ(landmark_2.location_normalized().width(), 0.07);
+  EXPECT_FLOAT_EQ(landmark_2.location_normalized().height(), 0.016666602);
+  EXPECT_FLOAT_EQ(landmark_2.score(), 0.00105);
+}
+
+TEST(FaceToRegionCalculatorTest, FaceScore) {
+  // Setup test
+  auto runner =
+      ::absl::make_unique<CalculatorRunner>(MakeConfig(true, false, false));
+  SetInputs(runner.get(), {kFace3});
+
+  // Run the calculator.
+  MP_ASSERT_OK(runner->Run());
+
+  // Check the output regions.
+  const std::vector<Packet>& output_packets =
+      runner->Outputs().Tag("REGIONS").packets;
+  ASSERT_EQ(1, output_packets.size());
+  const auto& regions = output_packets[0].Get<DetectionSet>();
+  ASSERT_EQ(1, regions.detections().size());
+  auto landmark_1 = regions.detections(0);
+  EXPECT_FLOAT_EQ(landmark_1.score(), 0.25);
+}
+
+}  // namespace
+}  // namespace autoflip
+}  // namespace mediapipe

mediapipe/examples/desktop/autoflip/calculators/localization_to_region_calculator.cc

@@ -0,0 +1,126 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <memory>
+#include <string>
+
+#include "absl/memory/memory.h"
+#include "mediapipe/examples/desktop/autoflip/autoflip_messages.pb.h"
+#include "mediapipe/examples/desktop/autoflip/calculators/localization_to_region_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/detection.pb.h"
+#include "mediapipe/framework/formats/location_data.pb.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+namespace autoflip {
+
+// This calculator converts detections from ObjectLocalizationCalculator to
+// SalientRegion protos that can be used for downstream processing.
+class LocalizationToRegionCalculator : public mediapipe::CalculatorBase {
+ public:
+  LocalizationToRegionCalculator();
+  ~LocalizationToRegionCalculator() override {}
+  LocalizationToRegionCalculator(const LocalizationToRegionCalculator&) =
+      delete;
+  LocalizationToRegionCalculator& operator=(
+      const LocalizationToRegionCalculator&) = delete;
+
+  static ::mediapipe::Status GetContract(mediapipe::CalculatorContract* cc);
+  ::mediapipe::Status Open(mediapipe::CalculatorContext* cc) override;
+  ::mediapipe::Status Process(mediapipe::CalculatorContext* cc) override;
+
+ private:
+  // Calculator options.
+  LocalizationToRegionCalculatorOptions options_;
+};
+REGISTER_CALCULATOR(LocalizationToRegionCalculator);
+
+LocalizationToRegionCalculator::LocalizationToRegionCalculator() {}
+
+namespace {
+
+// Converts an object detection to a autoflip SignalType.  Returns true if the
+// std::string label has a autoflip label.
+bool MatchType(const std::string& label, SignalType* type) {
+  if (label == "person") {
+    type->set_standard(SignalType::HUMAN);
+    return true;
+  }
+  if (label == "car" || label == "truck") {
+    type->set_standard(SignalType::CAR);
+    return true;
+  }
+  if (label == "dog" || label == "cat" || label == "bird" || label == "horse") {
+    type->set_standard(SignalType::PET);
+    return true;
+  }
+  return false;
+}
+
+// Converts a detection to a SalientRegion with a given label.
+void FillSalientRegion(const mediapipe::Detection& detection,
+                       const SignalType& label, SalientRegion* region) {
+  const auto& location = detection.location_data().relative_bounding_box();
+  region->mutable_location_normalized()->set_x(location.xmin());
+  region->mutable_location_normalized()->set_y(location.ymin());
+  region->mutable_location_normalized()->set_width(location.width());
+  region->mutable_location_normalized()->set_height(location.height());
+  region->set_score(1.0);
+  *region->mutable_signal_type() = label;
+}
+
+}  // namespace
+
+::mediapipe::Status LocalizationToRegionCalculator::GetContract(
+    mediapipe::CalculatorContract* cc) {
+  cc->Inputs().Tag("DETECTIONS").Set<std::vector<mediapipe::Detection>>();
+  cc->Outputs().Tag("REGIONS").Set<DetectionSet>();
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status LocalizationToRegionCalculator::Open(
+    mediapipe::CalculatorContext* cc) {
+  options_ = cc->Options<LocalizationToRegionCalculatorOptions>();
+
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status LocalizationToRegionCalculator::Process(
+    mediapipe::CalculatorContext* cc) {
+  const auto& annotations =
+      cc->Inputs().Tag("DETECTIONS").Get<std::vector<mediapipe::Detection>>();
+  auto regions = ::absl::make_unique<DetectionSet>();
+  for (const auto& detection : annotations) {
+    RET_CHECK_EQ(detection.label().size(), 1)
+        << "Number of labels not equal to one.";
+    SignalType autoflip_label;
+    if (MatchType(detection.label(0), &autoflip_label) &&
+        options_.output_standard_signals()) {
+      FillSalientRegion(detection, autoflip_label, regions->add_detections());
+    }
+    if (options_.output_all_signals()) {
+      SignalType object;
+      object.set_standard(SignalType::OBJECT);
+      FillSalientRegion(detection, object, regions->add_detections());
+    }
+  }
+
+  cc->Outputs().Tag("REGIONS").Add(regions.release(), cc->InputTimestamp());
+  return ::mediapipe::OkStatus();
+}
+
+}  // namespace autoflip
+}  // namespace mediapipe

mediapipe/examples/desktop/autoflip/calculators/localization_to_region_calculator.proto

@@ -0,0 +1,33 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe.autoflip;
+
+import "mediapipe/framework/calculator.proto";
+
+message LocalizationToRegionCalculatorOptions {
+  extend mediapipe.CalculatorOptions {
+    optional LocalizationToRegionCalculatorOptions ext = 284226721;
+  }
+
+  // Output standard autoflip signals only (Human, Pet, Car, etc) and apply
+  // standard autoflip labels.
+  optional bool output_standard_signals = 1 [default = true];
+  // Output all signals (regardless of label) and set autoflip label as
+  // 'Object'.  Can be combined with output_standard_signals giving each
+  // detection a 'object' label and a autoflip sepcific label.
+  optional bool output_all_signals = 2 [default = false];
+}

mediapipe/examples/desktop/autoflip/calculators/localization_to_region_calculator_test.cc

@@ -0,0 +1,164 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "absl/strings/string_view.h"
+#include "mediapipe/examples/desktop/autoflip/autoflip_messages.pb.h"
+#include "mediapipe/examples/desktop/autoflip/calculators/localization_to_region_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/formats/detection.pb.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/port/status_matchers.h"
+
+using mediapipe::Detection;
+
+namespace mediapipe {
+namespace autoflip {
+namespace {
+
+const char kConfig[] = R"(
+    calculator: "LocalizationToRegionCalculator"
+    input_stream: "DETECTIONS:detections"
+    output_stream: "REGIONS:regions"
+    )";
+
+const char kCar[] = R"(
+           label: "car"
+           location_data {
+           format: RELATIVE_BOUNDING_BOX
+           relative_bounding_box {
+             xmin: -0.00375
+             ymin: 0.003333
+             width: 0.125
+             height: 0.33333
+           }
+         })";
+
+const char kDog[] = R"(
+          label: "dog"
+           location_data {
+           format: RELATIVE_BOUNDING_BOX
+           relative_bounding_box {
+             xmin: 0.0025
+             ymin: 0.005
+             width: 0.25
+             height: 0.5
+           }
+         })";
+
+const char kZebra[] = R"(
+          label: "zebra"
+          location_data {
+           format: RELATIVE_BOUNDING_BOX
+           relative_bounding_box {
+             xmin: 0.0
+             ymin: 0.0
+             width: 0.5
+             height: 0.5
+           }
+         })";
+
+void SetInputs(CalculatorRunner* runner,
+               const std::vector<std::string>& detections) {
+  auto inputs = ::absl::make_unique<std::vector<Detection>>();
+  // A face with landmarks.
+  for (const auto& detection : detections) {
+    inputs->push_back(ParseTextProtoOrDie<Detection>(detection));
+  }
+  runner->MutableInputs()
+      ->Tag("DETECTIONS")
+      .packets.push_back(Adopt(inputs.release()).At(Timestamp::PostStream()));
+}
+
+CalculatorGraphConfig::Node MakeConfig(bool output_standard, bool output_all) {
+  auto config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfig);
+
+  config.mutable_options()
+      ->MutableExtension(LocalizationToRegionCalculatorOptions::ext)
+      ->set_output_standard_signals(output_standard);
+
+  config.mutable_options()
+      ->MutableExtension(LocalizationToRegionCalculatorOptions::ext)
+      ->set_output_all_signals(output_all);
+
+  return config;
+}
+
+TEST(LocalizationToRegionCalculatorTest, StandardTypes) {
+  // Setup test
+  auto runner = ::absl::make_unique<CalculatorRunner>(MakeConfig(true, false));
+  SetInputs(runner.get(), {kCar, kDog, kZebra});
+
+  // Run the calculator.
+  MP_ASSERT_OK(runner->Run());
+
+  // Check the output regions.
+  const std::vector<Packet>& output_packets =
+      runner->Outputs().Tag("REGIONS").packets;
+  ASSERT_EQ(1, output_packets.size());
+  const auto& regions = output_packets[0].Get<DetectionSet>();
+  ASSERT_EQ(2, regions.detections().size());
+  const auto& detection = regions.detections(0);
+  EXPECT_EQ(detection.signal_type().standard(), SignalType::CAR);
+  EXPECT_FLOAT_EQ(detection.location_normalized().x(), -0.00375);
+  EXPECT_FLOAT_EQ(detection.location_normalized().y(), 0.003333);
+  EXPECT_FLOAT_EQ(detection.location_normalized().width(), 0.125);
+  EXPECT_FLOAT_EQ(detection.location_normalized().height(), 0.33333);
+  const auto& detection_1 = regions.detections(1);
+  EXPECT_EQ(detection_1.signal_type().standard(), SignalType::PET);
+  EXPECT_FLOAT_EQ(detection_1.location_normalized().x(), 0.0025);
+  EXPECT_FLOAT_EQ(detection_1.location_normalized().y(), 0.005);
+  EXPECT_FLOAT_EQ(detection_1.location_normalized().width(), 0.25);
+  EXPECT_FLOAT_EQ(detection_1.location_normalized().height(), 0.5);
+}
+
+TEST(LocalizationToRegionCalculatorTest, AllTypes) {
+  // Setup test
+  auto runner = ::absl::make_unique<CalculatorRunner>(MakeConfig(false, true));
+  SetInputs(runner.get(), {kCar, kDog, kZebra});
+
+  // Run the calculator.
+  MP_ASSERT_OK(runner->Run());
+
+  // Check the output regions.
+  const std::vector<Packet>& output_packets =
+      runner->Outputs().Tag("REGIONS").packets;
+  ASSERT_EQ(1, output_packets.size());
+  const auto& regions = output_packets[0].Get<DetectionSet>();
+  ASSERT_EQ(3, regions.detections().size());
+}
+
+TEST(LocalizationToRegionCalculatorTest, BothTypes) {
+  // Setup test
+  auto runner = ::absl::make_unique<CalculatorRunner>(MakeConfig(true, true));
+  SetInputs(runner.get(), {kCar, kDog, kZebra});
+
+  // Run the calculator.
+  MP_ASSERT_OK(runner->Run());
+
+  // Check the output regions.
+  const std::vector<Packet>& output_packets =
+      runner->Outputs().Tag("REGIONS").packets;
+  ASSERT_EQ(1, output_packets.size());
+  const auto& regions = output_packets[0].Get<DetectionSet>();
+  ASSERT_EQ(5, regions.detections().size());
+}
+
+}  // namespace
+}  // namespace autoflip
+}  // namespace mediapipe

mediapipe/examples/desktop/autoflip/calculators/scene_cropping_calculator.cc

@@ -0,0 +1,589 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/examples/desktop/autoflip/calculators/scene_cropping_calculator.h"
+
+#include <cmath>
+
+#include "absl/memory/memory.h"
+#include "absl/strings/str_format.h"
+#include "mediapipe/examples/desktop/autoflip/autoflip_messages.pb.h"
+#include "mediapipe/examples/desktop/autoflip/quality/scene_cropping_viz.h"
+#include "mediapipe/examples/desktop/autoflip/quality/utils.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/image_frame.h"
+#include "mediapipe/framework/formats/image_frame_opencv.h"
+#include "mediapipe/framework/port/opencv_core_inc.h"
+#include "mediapipe/framework/port/opencv_imgproc_inc.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/timestamp.h"
+
+namespace mediapipe {
+namespace autoflip {
+
+constexpr char kInputVideoFrames[] = "VIDEO_FRAMES";
+constexpr char kInputKeyFrames[] = "KEY_FRAMES";
+constexpr char kInputDetections[] = "DETECTION_FEATURES";
+constexpr char kInputStaticFeatures[] = "STATIC_FEATURES";
+constexpr char kInputShotBoundaries[] = "SHOT_BOUNDARIES";
+constexpr char kInputExternalSettings[] = "EXTERNAL_SETTINGS";
+// This side packet must be used in conjunction with
+// TargetSizeType::MAXIMIZE_TARGET_DIMENSION
+constexpr char kAspectRatio[] = "EXTERNAL_ASPECT_RATIO";
+
+constexpr char kOutputCroppedFrames[] = "CROPPED_FRAMES";
+constexpr char kOutputKeyFrameCropViz[] = "KEY_FRAME_CROP_REGION_VIZ_FRAMES";
+constexpr char kOutputFocusPointFrameViz[] = "SALIENT_POINT_FRAME_VIZ_FRAMES";
+constexpr char kOutputSummary[] = "CROPPING_SUMMARY";
+
+::mediapipe::Status SceneCroppingCalculator::GetContract(
+    ::mediapipe::CalculatorContract* cc) {
+  if (cc->InputSidePackets().HasTag(kInputExternalSettings)) {
+    cc->InputSidePackets().Tag(kInputExternalSettings).Set<std::string>();
+  }
+  if (cc->InputSidePackets().HasTag(kAspectRatio)) {
+    cc->InputSidePackets().Tag(kAspectRatio).Set<std::string>();
+  }
+  cc->Inputs().Tag(kInputVideoFrames).Set<ImageFrame>();
+  if (cc->Inputs().HasTag(kInputKeyFrames)) {
+    cc->Inputs().Tag(kInputKeyFrames).Set<ImageFrame>();
+  }
+  cc->Inputs().Tag(kInputDetections).Set<DetectionSet>();
+  if (cc->Inputs().HasTag(kInputStaticFeatures)) {
+    cc->Inputs().Tag(kInputStaticFeatures).Set<StaticFeatures>();
+  }
+  cc->Inputs().Tag(kInputShotBoundaries).Set<bool>();
+
+  cc->Outputs().Tag(kOutputCroppedFrames).Set<ImageFrame>();
+  if (cc->Outputs().HasTag(kOutputKeyFrameCropViz)) {
+    cc->Outputs().Tag(kOutputKeyFrameCropViz).Set<ImageFrame>();
+  }
+  if (cc->Outputs().HasTag(kOutputFocusPointFrameViz)) {
+    cc->Outputs().Tag(kOutputFocusPointFrameViz).Set<ImageFrame>();
+  }
+  if (cc->Outputs().HasTag(kOutputSummary)) {
+    cc->Outputs().Tag(kOutputSummary).Set<VideoCroppingSummary>();
+  }
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status SceneCroppingCalculator::Open(CalculatorContext* cc) {
+  options_ = cc->Options<SceneCroppingCalculatorOptions>();
+  RET_CHECK_GT(options_.max_scene_size(), 0)
+      << "Maximum scene size is non-positive.";
+  RET_CHECK_GE(options_.prior_frame_buffer_size(), 0)
+      << "Prior frame buffer size is negative.";
+
+  RET_CHECK(options_.solid_background_frames_padding_fraction() >= 0.0 &&
+            options_.solid_background_frames_padding_fraction() <= 1.0)
+      << "Solid background frames padding fraction is not in [0, 1].";
+  const auto& padding_params = options_.padding_parameters();
+  background_contrast_ = padding_params.background_contrast();
+  RET_CHECK(background_contrast_ >= 0.0 && background_contrast_ <= 1.0)
+      << "Background contrast " << background_contrast_ << " is not in [0, 1].";
+  blur_cv_size_ = padding_params.blur_cv_size();
+  RET_CHECK_GT(blur_cv_size_, 0) << "Blur cv size is non-positive.";
+  overlay_opacity_ = padding_params.overlay_opacity();
+  RET_CHECK(overlay_opacity_ >= 0.0 && overlay_opacity_ <= 1.0)
+      << "Overlay opacity " << overlay_opacity_ << " is not in [0, 1].";
+
+  scene_cropper_ = absl::make_unique<SceneCropper>();
+  if (cc->Outputs().HasTag(kOutputSummary)) {
+    summary_ = absl::make_unique<VideoCroppingSummary>();
+  }
+  return ::mediapipe::OkStatus();
+}
+
+namespace {
+::mediapipe::Status ParseAspectRatioString(
+    const std::string& aspect_ratio_string, double* aspect_ratio) {
+  std::string error_msg =
+      "Aspect ratio std::string must be in the format of 'width:height', e.g. "
+      "'1:1' or '5:4', your input was " +
+      aspect_ratio_string;
+  auto pos = aspect_ratio_string.find(":");
+  RET_CHECK(pos != std::string::npos) << error_msg;
+  double width_ratio;
+  RET_CHECK(absl::SimpleAtod(aspect_ratio_string.substr(0, pos), &width_ratio))
+      << error_msg;
+  double height_ratio;
+  RET_CHECK(absl::SimpleAtod(
+      aspect_ratio_string.substr(pos + 1, aspect_ratio_string.size()),
+      &height_ratio))
+      << error_msg;
+  *aspect_ratio = width_ratio / height_ratio;
+  return ::mediapipe::OkStatus();
+}
+}  // namespace
+
+::mediapipe::Status SceneCroppingCalculator::Process(
+    ::mediapipe::CalculatorContext* cc) {
+  // Sets frame dimension and format.
+  if (frame_width_ < 0 &&
+      !cc->Inputs().Tag(kInputVideoFrames).Value().IsEmpty()) {
+    const auto& frame = cc->Inputs().Tag(kInputVideoFrames).Get<ImageFrame>();
+    frame_width_ = frame.Width();
+    RET_CHECK_GT(frame_width_, 0) << "Input frame width is non-positive.";
+    frame_height_ = frame.Height();
+    RET_CHECK_GT(frame_height_, 0) << "Input frame height is non-positive.";
+    frame_format_ = frame.Format();
+    target_width_ = options_.target_width();
+    target_height_ = options_.target_height();
+    if (cc->InputSidePackets().HasTag(kInputExternalSettings)) {
+      auto conversion_options = ParseTextProtoOrDie<ConversionOptions>(
+          cc->InputSidePackets()
+              .Tag(kInputExternalSettings)
+              .Get<std::string>());
+      target_width_ = conversion_options.target_width();
+      target_height_ = conversion_options.target_height();
+    }
+    target_aspect_ratio_ = static_cast<double>(target_width_) / target_height_;
+    RET_CHECK_NE(options_.target_size_type(),
+                 SceneCroppingCalculatorOptions::UNKNOWN)
+        << "TargetSizeType not set properly.";
+    // Resets target size if keep original height or width.
+    if (options_.target_size_type() ==
+        SceneCroppingCalculatorOptions::KEEP_ORIGINAL_HEIGHT) {
+      target_height_ = frame_height_;
+      target_width_ = std::round(target_height_ * target_aspect_ratio_);
+    } else if (options_.target_size_type() ==
+               SceneCroppingCalculatorOptions::KEEP_ORIGINAL_WIDTH) {
+      target_width_ = frame_width_;
+      target_height_ = std::round(target_width_ / target_aspect_ratio_);
+    } else if (options_.target_size_type() ==
+               SceneCroppingCalculatorOptions::MAXIMIZE_TARGET_DIMENSION) {
+      RET_CHECK(cc->InputSidePackets().HasTag(kAspectRatio))
+          << "MAXIMIZE_TARGET_DIMENSION is set without an "
+             "external_aspect_ratio";
+      double requested_aspect_ratio;
+      MP_RETURN_IF_ERROR(ParseAspectRatioString(
+          cc->InputSidePackets().Tag(kAspectRatio).Get<std::string>(),
+          &requested_aspect_ratio));
+      const double original_aspect_ratio =
+          static_cast<double>(frame_width_) / frame_height_;
+      if (original_aspect_ratio > requested_aspect_ratio) {
+        target_height_ = frame_height_;
+        target_width_ = std::round(target_height_ * requested_aspect_ratio);
+      } else {
+        target_width_ = frame_width_;
+        target_height_ = std::round(target_width_ / requested_aspect_ratio);
+      }
+    }
+    // Makes sure that target size is even if keep original width or height.
+    if (options_.target_size_type() !=
+        SceneCroppingCalculatorOptions::USE_TARGET_DIMENSION) {
+      if (target_width_ % 2 == 1) {
+        target_width_ = std::max(2, target_width_ - 1);
+      }
+      if (target_height_ % 2 == 1) {
+        target_height_ = std::max(2, target_height_ - 1);
+      }
+      target_aspect_ratio_ =
+          static_cast<double>(target_width_) / target_height_;
+    }
+    // Set keyframe width/height for feature upscaling (overwritten by keyframe
+    // input if provided).
+    if (options_.has_video_features_width() &&
+        options_.has_video_features_height()) {
+      key_frame_width_ = options_.video_features_width();
+      key_frame_height_ = options_.video_features_height();
+    } else if (!cc->Inputs().HasTag(kInputKeyFrames)) {
+      key_frame_width_ = frame_width_;
+      key_frame_height_ = frame_height_;
+    }
+    // Check provided dimensions.
+    RET_CHECK_GT(target_width_, 0) << "Target width is non-positive.";
+    RET_CHECK_NE(target_width_ % 2, 1)
+        << "Target width cannot be odd, because encoder expects dimension "
+           "values to be even.";
+    RET_CHECK_GT(target_height_, 0) << "Target height is non-positive.";
+    RET_CHECK_NE(target_height_ % 2, 1)
+        << "Target height cannot be odd, because encoder expects dimension "
+           "values to be even.";
+  }
+
+  // Sets key frame dimension.
+  if (cc->Inputs().HasTag(kInputKeyFrames) &&
+      !cc->Inputs().Tag(kInputKeyFrames).Value().IsEmpty() &&
+      key_frame_width_ < 0) {
+    const auto& key_frame = cc->Inputs().Tag(kInputKeyFrames).Get<ImageFrame>();
+    key_frame_width_ = key_frame.Width();
+    key_frame_height_ = key_frame.Height();
+  }
+
+  // Processes a scene when shot boundary or buffer is full.
+  bool is_end_of_scene = false;
+  if (!cc->Inputs().Tag(kInputShotBoundaries).Value().IsEmpty()) {
+    is_end_of_scene = cc->Inputs().Tag(kInputShotBoundaries).Get<bool>();
+  }
+  const bool force_buffer_flush =
+      scene_frames_.size() >= options_.max_scene_size();
+  if (!scene_frames_.empty() && (is_end_of_scene || force_buffer_flush)) {
+    MP_RETURN_IF_ERROR(ProcessScene(is_end_of_scene, cc));
+  }
+
+  // Saves frame and timestamp and whether it is a key frame.
+  if (!cc->Inputs().Tag(kInputVideoFrames).Value().IsEmpty()) {
+    LOG_EVERY_N(ERROR, 10)
+        << "------------------------ (Breathing) Time(s): "
+        << cc->Inputs().Tag(kInputVideoFrames).Value().Timestamp().Seconds();
+    const auto& frame = cc->Inputs().Tag(kInputVideoFrames).Get<ImageFrame>();
+    const cv::Mat frame_mat = formats::MatView(&frame);
+    cv::Mat copy_mat;
+    frame_mat.copyTo(copy_mat);
+    scene_frames_.push_back(copy_mat);
+    scene_frame_timestamps_.push_back(cc->InputTimestamp().Value());
+    is_key_frames_.push_back(
+        !cc->Inputs().Tag(kInputDetections).Value().IsEmpty());
+  }
+
+  // Packs key frame info.
+  if (!cc->Inputs().Tag(kInputDetections).Value().IsEmpty()) {
+    const auto& detections =
+        cc->Inputs().Tag(kInputDetections).Get<DetectionSet>();
+    KeyFrameInfo key_frame_info;
+    MP_RETURN_IF_ERROR(PackKeyFrameInfo(
+        cc->InputTimestamp().Value(), detections, frame_width_, frame_height_,
+        key_frame_width_, key_frame_height_, &key_frame_info));
+    key_frame_infos_.push_back(key_frame_info);
+  }
+
+  // Buffers static features.
+  if (cc->Inputs().HasTag(kInputStaticFeatures) &&
+      !cc->Inputs().Tag(kInputStaticFeatures).Value().IsEmpty()) {
+    static_features_.push_back(
+        cc->Inputs().Tag(kInputStaticFeatures).Get<StaticFeatures>());
+    static_features_timestamps_.push_back(cc->InputTimestamp().Value());
+  }
+
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status SceneCroppingCalculator::Close(
+    ::mediapipe::CalculatorContext* cc) {
+  if (!scene_frames_.empty()) {
+    MP_RETURN_IF_ERROR(ProcessScene(/* is_end_of_scene = */ true, cc));
+  }
+  if (cc->Outputs().HasTag(kOutputSummary)) {
+    cc->Outputs()
+        .Tag(kOutputSummary)
+        .Add(summary_.release(), Timestamp::PostStream());
+  }
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status SceneCroppingCalculator::RemoveStaticBorders() {
+  int top_border_size = 0, bottom_border_size = 0;
+  MP_RETURN_IF_ERROR(ComputeSceneStaticBordersSize(
+      static_features_, &top_border_size, &bottom_border_size));
+  const double scale = static_cast<double>(frame_height_) / key_frame_height_;
+  top_border_distance_ = std::round(scale * top_border_size);
+  const int bottom_border_distance = std::round(scale * bottom_border_size);
+  effective_frame_height_ =
+      frame_height_ - top_border_distance_ - bottom_border_distance;
+
+  if (top_border_distance_ > 0 || bottom_border_distance > 0) {
+    VLOG(1) << "Remove top border " << top_border_distance_ << " bottom border "
+            << bottom_border_distance;
+    // Remove borders from frames.
+    cv::Rect roi(0, top_border_distance_, frame_width_,
+                 effective_frame_height_);
+    for (int i = 0; i < scene_frames_.size(); ++i) {
+      cv::Mat tmp;
+      scene_frames_[i](roi).copyTo(tmp);
+      scene_frames_[i] = tmp;
+    }
+    // Adjust detection bounding boxes.
+    for (int i = 0; i < key_frame_infos_.size(); ++i) {
+      DetectionSet adjusted_detections;
+      const auto& detections = key_frame_infos_[i].detections();
+      for (int j = 0; j < detections.detections_size(); ++j) {
+        const auto& detection = detections.detections(j);
+        SalientRegion adjusted_detection = detection;
+        // Clamp the box to be within the de-bordered frame.
+        if (!ClampRect(0, top_border_distance_, frame_width_,
+                       top_border_distance_ + effective_frame_height_,
+                       adjusted_detection.mutable_location())
+                 .ok()) {
+          continue;
+        }
+        // Offset the y position.
+        adjusted_detection.mutable_location()->set_y(
+            adjusted_detection.location().y() - top_border_distance_);
+        *adjusted_detections.add_detections() = adjusted_detection;
+      }
+      *key_frame_infos_[i].mutable_detections() = adjusted_detections;
+    }
+  }
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status
+SceneCroppingCalculator::InitializeFrameCropRegionComputer() {
+  key_frame_crop_options_ = options_.key_frame_crop_options();
+  MP_RETURN_IF_ERROR(
+      SetKeyFrameCropTarget(frame_width_, effective_frame_height_,
+                            target_aspect_ratio_, &key_frame_crop_options_));
+  VLOG(1) << "Target width " << key_frame_crop_options_.target_width();
+  VLOG(1) << "Target height " << key_frame_crop_options_.target_height();
+  frame_crop_region_computer_ =
+      absl::make_unique<FrameCropRegionComputer>(key_frame_crop_options_);
+  return ::mediapipe::OkStatus();
+}
+
+void SceneCroppingCalculator::FilterKeyFrameInfo() {
+  if (!options_.user_hint_override()) {
+    return;
+  }
+  std::vector<KeyFrameInfo> user_hints_only;
+  bool has_user_hints = false;
+  for (auto key_frame : key_frame_infos_) {
+    DetectionSet user_hint_only_set;
+    for (const auto& detection : key_frame.detections().detections()) {
+      if (detection.signal_type().has_standard() &&
+          detection.signal_type().standard() == SignalType::USER_HINT) {
+        *user_hint_only_set.add_detections() = detection;
+        has_user_hints = true;
+      }
+    }
+    *key_frame.mutable_detections() = user_hint_only_set;
+    user_hints_only.push_back(key_frame);
+  }
+  if (has_user_hints) {
+    key_frame_infos_ = user_hints_only;
+  }
+}
+
+::mediapipe::Status SceneCroppingCalculator::ProcessScene(
+    const bool is_end_of_scene, CalculatorContext* cc) {
+  // Removes detections under special circumstances.
+  FilterKeyFrameInfo();
+
+  // Removes any static borders.
+  MP_RETURN_IF_ERROR(RemoveStaticBorders());
+
+  // Decides if solid background color padding is possible and sets up color
+  // interpolation functions in CIELAB. Uses linear interpolation by default.
+  MP_RETURN_IF_ERROR(FindSolidBackgroundColor(
+      static_features_, static_features_timestamps_,
+      options_.solid_background_frames_padding_fraction(),
+      &has_solid_background_, &background_color_l_function_,
+      &background_color_a_function_, &background_color_b_function_));
+
+  // Computes key frame crop regions.
+  MP_RETURN_IF_ERROR(InitializeFrameCropRegionComputer());
+  const int num_key_frames = key_frame_infos_.size();
+  std::vector<KeyFrameCropResult> key_frame_crop_results(num_key_frames);
+  for (int i = 0; i < num_key_frames; ++i) {
+    MP_RETURN_IF_ERROR(frame_crop_region_computer_->ComputeFrameCropRegion(
+        key_frame_infos_[i], &key_frame_crop_results[i]));
+  }
+
+  // Analyzes scene camera motion and generates FocusPointFrames.
+  auto analyzer_options = options_.scene_camera_motion_analyzer_options();
+  analyzer_options.set_allow_sweeping(analyzer_options.allow_sweeping() &&
+                                      !has_solid_background_);
+  scene_camera_motion_analyzer_ =
+      absl::make_unique<SceneCameraMotionAnalyzer>(analyzer_options);
+  SceneKeyFrameCropSummary scene_summary;
+  std::vector<FocusPointFrame> focus_point_frames;
+  SceneCameraMotion scene_camera_motion;
+  MP_RETURN_IF_ERROR(
+      scene_camera_motion_analyzer_->AnalyzeSceneAndPopulateFocusPointFrames(
+          key_frame_infos_, key_frame_crop_options_, key_frame_crop_results,
+          frame_width_, effective_frame_height_, scene_frame_timestamps_,
+          &scene_summary, &focus_point_frames, &scene_camera_motion));
+
+  // Crops scene frames.
+  std::vector<cv::Mat> cropped_frames;
+  MP_RETURN_IF_ERROR(scene_cropper_->CropFrames(
+      scene_summary, scene_frames_, focus_point_frames,
+      prior_focus_point_frames_, &cropped_frames));
+
+  // Formats and outputs cropped frames.
+  bool apply_padding = false;
+  float vertical_fill_precent;
+  MP_RETURN_IF_ERROR(FormatAndOutputCroppedFrames(
+      cropped_frames, &apply_padding, &vertical_fill_precent, cc));
+
+  // Caches prior FocusPointFrames if this was not the end of a scene.
+  prior_focus_point_frames_.clear();
+  if (!is_end_of_scene) {
+    const int start = std::max(0, static_cast<int>(scene_frames_.size()) -
+                                      options_.prior_frame_buffer_size());
+    for (int i = start; i < num_key_frames; ++i) {
+      prior_focus_point_frames_.push_back(focus_point_frames[i]);
+    }
+  }
+
+  // Optionally outputs visualization frames.
+  MP_RETURN_IF_ERROR(OutputVizFrames(key_frame_crop_results, focus_point_frames,
+                                     scene_summary.crop_window_width(),
+                                     scene_summary.crop_window_height(), cc));
+
+  const double start_sec = Timestamp(scene_frame_timestamps_.front()).Seconds();
+  const double end_sec = Timestamp(scene_frame_timestamps_.back()).Seconds();
+  VLOG(1) << absl::StrFormat("Processed a scene from %.2f sec to %.2f sec",
+                             start_sec, end_sec);
+
+  // Optionally makes summary.
+  if (cc->Outputs().HasTag(kOutputSummary)) {
+    auto* scene_summary = summary_->add_scene_summaries();
+    scene_summary->set_start_sec(start_sec);
+    scene_summary->set_end_sec(end_sec);
+    *(scene_summary->mutable_camera_motion()) = scene_camera_motion;
+    scene_summary->set_is_end_of_scene(is_end_of_scene);
+    scene_summary->set_is_padded(apply_padding);
+  }
+
+  key_frame_infos_.clear();
+  scene_frames_.clear();
+  scene_frame_timestamps_.clear();
+  is_key_frames_.clear();
+  static_features_.clear();
+  static_features_timestamps_.clear();
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status SceneCroppingCalculator::FormatAndOutputCroppedFrames(
+    const std::vector<cv::Mat>& cropped_frames, bool* apply_padding,
+    float* vertical_fill_precent, CalculatorContext* cc) {
+  RET_CHECK(apply_padding) << "Has padding boolean is null.";
+  if (cropped_frames.empty()) {
+    return ::mediapipe::OkStatus();
+  }
+
+  // Computes scaling factor and decides if padding is needed.
+  const int crop_width = cropped_frames.front().cols;
+  const int crop_height = cropped_frames.front().rows;
+  VLOG(1) << "crop_width = " << crop_width << " crop_height = " << crop_height;
+  const double scaling =
+      std::max(static_cast<double>(target_width_) / crop_width,
+               static_cast<double>(target_height_) / crop_height);
+  int scaled_width = std::round(scaling * crop_width);
+  int scaled_height = std::round(scaling * crop_height);
+  RET_CHECK_GE(scaled_width, target_width_)
+      << "Scaled width is less than target width - something is wrong.";
+  RET_CHECK_GE(scaled_height, target_height_)
+      << "Scaled height is less than target height - something is wrong.";
+  if (scaled_width - target_width_ <= 1) scaled_width = target_width_;
+  if (scaled_height - target_height_ <= 1) scaled_height = target_height_;
+  *apply_padding =
+      scaled_width != target_width_ || scaled_height != target_height_;
+  *vertical_fill_precent = scaled_height / static_cast<float>(target_height_);
+  if (*apply_padding) {
+    padder_ = absl::make_unique<PaddingEffectGenerator>(
+        scaled_width, scaled_height, target_aspect_ratio_);
+    VLOG(1) << "Scene is padded: scaled width = " << scaled_width
+            << " target width = " << target_width_
+            << " scaled height = " << scaled_height
+            << " target height = " << target_height_;
+  }
+
+  // Resizes cropped frames, pads frames, and output frames.
+  cv::Scalar* background_color = nullptr;
+  cv::Scalar interpolated_color;
+  const int num_frames = cropped_frames.size();
+  for (int i = 0; i < num_frames; ++i) {
+    const int64 time_ms = scene_frame_timestamps_[i];
+    const Timestamp timestamp(time_ms);
+    auto scaled_frame = absl::make_unique<ImageFrame>(
+        frame_format_, scaled_width, scaled_height);
+    auto destination = formats::MatView(scaled_frame.get());
+    if (scaled_width == crop_width && scaled_height == crop_height) {
+      cropped_frames[i].copyTo(destination);
+    } else {
+      // cubic is better quality for upscaling and area is good for downscaling
+      const int interpolation_method =
+          scaling > 1 ? cv::INTER_CUBIC : cv::INTER_AREA;
+      cv::resize(cropped_frames[i], destination, destination.size(), 0, 0,
+                 interpolation_method);
+    }
+    if (*apply_padding) {
+      if (has_solid_background_) {
+        double lab[3];
+        lab[0] = background_color_l_function_.Evaluate(time_ms);
+        lab[1] = background_color_a_function_.Evaluate(time_ms);
+        lab[2] = background_color_b_function_.Evaluate(time_ms);
+        cv::Mat3f lab_mat(1, 1, cv::Vec3f(lab[0], lab[1], lab[2]));
+        cv::Mat3f rgb_mat(1, 1);
+        // Necessary scaling of the RGB values from [0, 1] to [0, 255] based on:
+        // https://docs.opencv.org/2.4/modules/imgproc/doc/miscellaneous_transformations.html#cvtcolor
+        cv::cvtColor(lab_mat, rgb_mat, cv::COLOR_Lab2RGB);
+        rgb_mat *= 255.0;
+        auto k = rgb_mat.at<cv::Vec3f>(0, 0);
+        k[0] = k[0] < 0.0 ? 0.0 : k[0] > 255.0 ? 255.0 : k[0];
+        k[1] = k[1] < 0.0 ? 0.0 : k[1] > 255.0 ? 255.0 : k[1];
+        k[2] = k[2] < 0.0 ? 0.0 : k[2] > 255.0 ? 255.0 : k[2];
+        interpolated_color =
+            cv::Scalar(std::round(k[0]), std::round(k[1]), std::round(k[2]));
+        background_color = &interpolated_color;
+      }
+      auto padded_frame = absl::make_unique<ImageFrame>();
+      MP_RETURN_IF_ERROR(padder_->Process(
+          *scaled_frame, background_contrast_,
+          std::min({blur_cv_size_, scaled_width, scaled_height}),
+          overlay_opacity_, padded_frame.get(), background_color));
+      RET_CHECK_EQ(padded_frame->Width(), target_width_)
+          << "Padded frame width is off.";
+      RET_CHECK_EQ(padded_frame->Height(), target_height_)
+          << "Padded frame height is off.";
+      cc->Outputs()
+          .Tag(kOutputCroppedFrames)
+          .Add(padded_frame.release(), timestamp);
+    } else {
+      cc->Outputs()
+          .Tag(kOutputCroppedFrames)
+          .Add(scaled_frame.release(), timestamp);
+    }
+  }
+  return ::mediapipe::OkStatus();
+}
+
+mediapipe::Status SceneCroppingCalculator::OutputVizFrames(
+    const std::vector<KeyFrameCropResult>& key_frame_crop_results,
+    const std::vector<FocusPointFrame>& focus_point_frames,
+    const int crop_window_width, const int crop_window_height,
+    CalculatorContext* cc) const {
+  if (cc->Outputs().HasTag(kOutputKeyFrameCropViz)) {
+    std::vector<std::unique_ptr<ImageFrame>> viz_frames;
+    MP_RETURN_IF_ERROR(DrawDetectionsAndCropRegions(
+        scene_frames_, is_key_frames_, key_frame_infos_, key_frame_crop_results,
+        frame_format_, &viz_frames));
+    for (int i = 0; i < scene_frames_.size(); ++i) {
+      cc->Outputs()
+          .Tag(kOutputKeyFrameCropViz)
+          .Add(viz_frames[i].release(), Timestamp(scene_frame_timestamps_[i]));
+    }
+  }
+  if (cc->Outputs().HasTag(kOutputFocusPointFrameViz)) {
+    std::vector<std::unique_ptr<ImageFrame>> viz_frames;
+    MP_RETURN_IF_ERROR(DrawFocusPointAndCropWindow(
+        scene_frames_, focus_point_frames, options_.viz_overlay_opacity(),
+        crop_window_width, crop_window_height, frame_format_, &viz_frames));
+    for (int i = 0; i < scene_frames_.size(); ++i) {
+      cc->Outputs()
+          .Tag(kOutputFocusPointFrameViz)
+          .Add(viz_frames[i].release(), Timestamp(scene_frame_timestamps_[i]));
+    }
+  }
+  return ::mediapipe::OkStatus();
+}
+
+REGISTER_CALCULATOR(SceneCroppingCalculator);
+
+}  // namespace autoflip
+}  // namespace mediapipe

mediapipe/examples/desktop/autoflip/calculators/scene_cropping_calculator.h

@@ -0,0 +1,249 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_CALCULATORS_SCENE_CROPPING_CALCULATOR_H_
+#define MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_CALCULATORS_SCENE_CROPPING_CALCULATOR_H_
+
+#include <memory>
+#include <vector>
+
+#include "mediapipe/examples/desktop/autoflip/autoflip_messages.pb.h"
+#include "mediapipe/examples/desktop/autoflip/calculators/scene_cropping_calculator.pb.h"
+#include "mediapipe/examples/desktop/autoflip/quality/cropping.pb.h"
+#include "mediapipe/examples/desktop/autoflip/quality/focus_point.pb.h"
+#include "mediapipe/examples/desktop/autoflip/quality/frame_crop_region_computer.h"
+#include "mediapipe/examples/desktop/autoflip/quality/padding_effect_generator.h"
+#include "mediapipe/examples/desktop/autoflip/quality/piecewise_linear_function.h"
+#include "mediapipe/examples/desktop/autoflip/quality/polynomial_regression_path_solver.h"
+#include "mediapipe/examples/desktop/autoflip/quality/scene_camera_motion_analyzer.h"
+#include "mediapipe/examples/desktop/autoflip/quality/scene_cropper.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/image_frame.h"
+#include "mediapipe/framework/port/opencv_core_inc.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+namespace autoflip {
+// This calculator crops video scenes to target size, which can be of any aspect
+// ratio. The calculator supports both "landscape -> portrait", and "portrait ->
+// landscape" use cases. The two use cases are automatically determined by
+// comparing the input and output frame's aspect ratios internally.
+//
+// The target (i.e. output) frame's dimension can be specified through the
+// target_width(height) fields in the options. Both this target dimension and
+// the input dimension should be even. If either keep_original_height or
+// keep_original_width is set to true, the corresponding target dimension will
+// only be used to compute the aspect ratio (as opposed to setting the actual
+// dimension) of the output. If the output frame thus computed has an odd
+// size, it will be rounded down to an even number.
+//
+// The calculator takes shot boundary signals to identify shot boundaries, and
+// crops each scene independently. The cropping decisions are made based on
+// detection features, which are a collection of focus regions detected from
+// different signals, and then fused together by a SignalFusingCalculator. To
+// add a new type of focus signals, it should be added in the input of the
+// SignalFusingCalculator, which can take an arbitrary number of input streams.
+//
+// If after attempting to cover focus regions based on the cropping decisions
+// made, the retained frame region's aspect ratio is still different from the
+// target aspect ratio, padding will be applied. In this case, a seamless
+// padding with a solid color would be preferred wherever possible, given
+// information from the input static features; otherwise, a simple padding with
+// centered foreground on blurred background will be applied.
+//
+// The main complexity of this calculator lies in stabilizing crop regions over
+// the scene using a Retargeter, which solves linear programming problems
+// through a L1 path solver (default) or least squares problems through a L2
+// path solver.
+
+// Input streams:
+// - required tag VIDEO_FRAMES (type ImageFrame):
+//     Original scene frames to be cropped.
+// - required tag DETECTION_FEATURES (type DetectionSet):
+//     Detected features on the key frames.
+// - optional tag STATIC_FEATURES (type StaticFeatures):
+//     Detected features on the key frames.
+// - required tag SHOT_BOUNDARIES (type bool):
+//     Indicators for shot boundaries (output of shot boundary detection).
+// - optional tag KEY_FRAMES (type ImageFrame):
+//     Key frames on which features are detected. This is only used to set the
+//     detection features frame size, and when it is omitted, the features frame
+//     size is assumed to be the original scene frame size.
+//
+// Output streams:
+// - required tag CROPPED_FRAMES (type ImageFrame):
+//     Cropped frames at target size and original frame rate.
+// - optional tag KEY_FRAME_CROP_REGION_VIZ_FRAMES (type ImageFrame):
+//     Debug visualization frames at original frame size and frame rate. Draws
+//     the required (yellow) and non-required (cyan) detection features and the
+//     key frame crop regions (green).
+// - optional tag SALIENT_POINT_FRAME_VIZ_FRAMES (type ImageFrame):
+//     Debug visualization frames at original frame size and frame rate. Draws
+//     the focus points and the scene crop window (red).
+// - optional tag CROPPING_SUMMARY (type VideoCroppingSummary):
+//     Debug summary information for the video. Only generates one packet when
+//     calculator closes.
+//
+// Example config:
+// node {
+//   calculator: "SceneCroppingCalculator"
+//   input_stream: "VIDEO_FRAMES:camera_frames_org"
+//   input_stream: "KEY_FRAMES:down_sampled_frames"
+//   input_stream: "DETECTION_FEATURES:focus_regions"
+//   input_stream: "STATIC_FEATURES:border_features"
+//   input_stream: "SHOT_BOUNDARIES:shot_boundary_frames"
+//   output_stream: "CROPPED_FRAMES:cropped_frames"
+//   options: {
+//     [mediapipe.SceneCroppingCalculatorOptions.ext]: {
+//       target_width: 720
+//       target_height: 1124
+//       target_size_type: USE_TARGET_DIMENSION
+//     }
+//   }
+// }
+// Note that only the target size is required in the options, and all other
+// fields are optional with default settings.
+class SceneCroppingCalculator : public CalculatorBase {
+ public:
+  static ::mediapipe::Status GetContract(CalculatorContract* cc);
+
+  // Validates calculator options and initializes SceneCameraMotionAnalyzer and
+  // SceneCropper.
+  ::mediapipe::Status Open(CalculatorContext* cc) override;
+
+  // Buffers each scene frame and its timestamp. Packs and stores KeyFrameInfo
+  // for key frames (a.k.a. frames with detection features). When a shot
+  // boundary is encountered or when the buffer is full, calls ProcessScene()
+  // to process the scene at once, and clears buffers.
+  ::mediapipe::Status Process(CalculatorContext* cc) override;
+
+  // Calls ProcessScene() on remaining buffered frames. Optionally outputs a
+  // VideoCroppingSummary if the output stream CROPPING_SUMMARY is present.
+  ::mediapipe::Status Close(::mediapipe::CalculatorContext* cc) override;
+
+ private:
+  // Removes any static borders from the scene frames before cropping.
+  ::mediapipe::Status RemoveStaticBorders();
+
+  // Initializes a FrameCropRegionComputer given input and target frame sizes.
+  ::mediapipe::Status InitializeFrameCropRegionComputer();
+
+  // Processes a scene using buffered scene frames and KeyFrameInfos:
+  // 1. Computes key frame crop regions using a FrameCropRegionComputer.
+  // 2. Analyzes scene camera motion and generates FocusPointFrames using a
+  //    SceneCameraMotionAnalyzer.
+  // 3. Crops scene frames using a SceneCropper (wrapper around Retargeter).
+  // 4. Formats and outputs cropped frames .
+  // 5. Caches prior FocusPointFrames if this is not the end of a scene (due
+  //    to force flush).
+  // 6. Optionally outputs visualization frames.
+  // 7. Optionally updates cropping summary.
+  ::mediapipe::Status ProcessScene(const bool is_end_of_scene,
+                                   CalculatorContext* cc);
+
+  // Formats and outputs the cropped frames. Scales them to be at least as big
+  // as the target size. If the aspect ratio is different, applies padding. Uses
+  // solid background from static features if possible, otherwise uses blurred
+  // background. Sets apply_padding to true if the scene is padded.
+  ::mediapipe::Status FormatAndOutputCroppedFrames(
+      const std::vector<cv::Mat>& cropped_frames, bool* apply_padding,
+      float* vertical_fill_precent, CalculatorContext* cc);
+
+  // Draws and outputs visualization frames if those streams are present.
+  ::mediapipe::Status OutputVizFrames(
+      const std::vector<KeyFrameCropResult>& key_frame_crop_results,
+      const std::vector<FocusPointFrame>& focus_point_frames,
+      const int crop_window_width, const int crop_window_height,
+      CalculatorContext* cc) const;
+
+  // Filters detections based on USER_HINT under specific flag conditions.
+  void FilterKeyFrameInfo();
+
+  // Target frame size and aspect ratio passed in or computed from options.
+  int target_width_ = -1;
+  int target_height_ = -1;
+  double target_aspect_ratio_ = -1.0;
+
+  // Input video frame size and format.
+  int frame_width_ = -1;
+  int frame_height_ = -1;
+  ImageFormat::Format frame_format_ = ImageFormat::UNKNOWN;
+
+  // Key frame size (frame size for detections and border detections).
+  int key_frame_width_ = -1;
+  int key_frame_height_ = -1;
+
+  // Calculator options.
+  SceneCroppingCalculatorOptions options_;
+
+  // Buffered KeyFrameInfos for the current scene (size = number of key frames).
+  std::vector<KeyFrameInfo> key_frame_infos_;
+
+  // Buffered frames, timestamps, and indicators for key frames in the current
+  // scene (size = number of input video frames).
+  std::vector<cv::Mat> scene_frames_;
+  std::vector<int64> scene_frame_timestamps_;
+  std::vector<bool> is_key_frames_;
+
+  // Static border information for the scene.
+  int top_border_distance_ = -1;
+  int effective_frame_height_ = -1;
+
+  // Stored FocusPointFrames from prior scene when there was no actual scene
+  // change (due to forced flush when buffer is full).
+  std::vector<FocusPointFrame> prior_focus_point_frames_;
+
+  // KeyFrameCropOptions used by the FrameCropRegionComputer.
+  KeyFrameCropOptions key_frame_crop_options_;
+
+  // Object for computing key frame crop regions from detection features.
+  std::unique_ptr<FrameCropRegionComputer> frame_crop_region_computer_ =
+      nullptr;
+
+  // Object for analyzing scene camera motion from key frame crop regions and
+  // generating FocusPointFrames.
+  std::unique_ptr<SceneCameraMotionAnalyzer> scene_camera_motion_analyzer_ =
+      nullptr;
+
+  // Object for cropping a scene given FocusPointFrames.
+  std::unique_ptr<SceneCropper> scene_cropper_ = nullptr;
+
+  // Buffered static features and their timestamps used in padding with solid
+  // background color (size = number of frames with static features).
+  std::vector<StaticFeatures> static_features_;
+  std::vector<int64> static_features_timestamps_;
+  bool has_solid_background_ = false;
+  // CIELAB yields more natural color transitions than RGB and HSV: RGB tends to
+  // produce darker in-between colors and HSV can introduce new hues. See
+  // https://howaboutanorange.com/blog/2011/08/10/color_interpolation/ for
+  // visual comparisons of color transition in different spaces.
+  PiecewiseLinearFunction background_color_l_function_;  // CIELAB - l
+  PiecewiseLinearFunction background_color_a_function_;  // CIELAB - a
+  PiecewiseLinearFunction background_color_b_function_;  // CIELAB - b
+
+  // Parameters for padding with blurred background passed in from options.
+  float background_contrast_ = -1.0;
+  int blur_cv_size_ = -1;
+  float overlay_opacity_ = -1.0;
+  // Object for padding an image to a target aspect ratio.
+  std::unique_ptr<PaddingEffectGenerator> padder_ = nullptr;
+
+  // Optional diagnostic summary output emitted in Close().
+  std::unique_ptr<VideoCroppingSummary> summary_ = nullptr;
+};
+}  // namespace autoflip
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_CALCULATORS_SCENE_CROPPING_CALCULATOR_H_

mediapipe/examples/desktop/autoflip/calculators/scene_cropping_calculator.proto

@@ -0,0 +1,101 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe.autoflip;
+
+import "mediapipe/examples/desktop/autoflip/quality/cropping.proto";
+import "mediapipe/framework/calculator.proto";
+
+// Options for the SceneCroppingCalculator.
+message SceneCroppingCalculatorOptions {
+  extend mediapipe.CalculatorOptions {
+    optional SceneCroppingCalculatorOptions ext = 284806831;
+  }
+
+  // Target frame size - this has to be even (for ffmpeg encoding).
+  optional int32 target_width = 1;
+  optional int32 target_height = 2;
+
+  // Choices for target size specification.
+  enum TargetSizeType {
+    // Unknown type (needed by ProtoBestPractices to ensure consistent behavior
+    // across proto2 and proto3). This type should not be used.
+    UNKNOWN = 0;
+    // Directly uses the target dimension given above.
+    USE_TARGET_DIMENSION = 1;
+    // Uses the target dimension to compute the target aspect ratio, but keeps
+    // original height/width. If the resulting size for the other dimension is
+    // odd, it is rounded down to an even size.
+    KEEP_ORIGINAL_HEIGHT = 2;
+    KEEP_ORIGINAL_WIDTH = 3;
+    // Used on conjuntion with external_aspect_ratio, create the largest sized
+    // output without upscaling the video.
+    MAXIMIZE_TARGET_DIMENSION = 4;
+  }
+  optional TargetSizeType target_size_type = 3 [default = USE_TARGET_DIMENSION];
+
+  // Forces a flush of the frame buffer after this number of frames even if
+  // there is not a shot boundary.
+  optional int32 max_scene_size = 4 [default = 600];
+
+  // Number of frames from prior buffer to be used to smooth out camera
+  // trajectory when it was a forced flush.
+  optional int32 prior_frame_buffer_size = 5 [default = 30];
+
+  // Options for computing key frame crop regions using the
+  // FrameCropRegionComputer.
+  // **** Note: You shall NOT manually set the target width and height fields
+  // inside this field as they will be overridden internally in the calculator
+  // (i.e. automatically computed from target aspect ratio).
+  optional KeyFrameCropOptions key_frame_crop_options = 6;
+
+  // Options for analyzing scene camera motion and populating SalientPointFrames
+  // using the SceneCameraMotionAnalyzer.
+  optional SceneCameraMotionAnalyzerOptions
+      scene_camera_motion_analyzer_options = 7;
+
+  // If the fraction of frames with solid background in one shot exceeds this
+  // threshold, use a solid color for background in padding for this shot.
+  optional float solid_background_frames_padding_fraction = 8 [default = 0.6];
+
+  // Options for padding using the PaddingEffectGenerator (copied from
+  // ad_creation/calculators/universal_padding_calculator.proto).
+  message PaddingEffectParameters {
+    // Contrast adjustment for padding background. This value should between 0
+    // and 1. The smaller the value, the darker the background. 1 means no
+    // contrast change.
+    optional float background_contrast = 1 [default = 1.0];
+    // The cv::Size() parameter used in creating blurry effects for padding
+    // backgrounds.
+    optional int32 blur_cv_size = 2 [default = 200];
+    // The opacity of the black layer overlaied on top of the background. The
+    // value should be within [0, 1], in which 0 means totally transparent, and
+    // 1 means totally opaque.
+    optional float overlay_opacity = 3 [default = 0.6];
+  }
+  optional PaddingEffectParameters padding_parameters = 9;
+
+  // If set and input "KEY_FRAMES" not provided, uses these keyframe values.
+  optional int32 video_features_width = 10;
+  optional int32 video_features_height = 11;
+
+  // If a user hint is provided on a scene, use only this signal for cropping
+  // and camera motion.
+  optional bool user_hint_override = 12;
+
+  // An opacity used to render cropping windows for visualization purposes.
+  optional float viz_overlay_opacity = 13 [default = 0.7];
+}

mediapipe/examples/desktop/autoflip/calculators/scene_cropping_calculator_test.cc

@@ -0,0 +1,621 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/examples/desktop/autoflip/calculators/scene_cropping_calculator.h"
+
+#include <random>
+#include <utility>
+#include <vector>
+
+#include "mediapipe/examples/desktop/autoflip/autoflip_messages.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/formats/image_frame_opencv.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/opencv_core_inc.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/port/status_matchers.h"
+
+namespace mediapipe {
+namespace autoflip {
+namespace {
+
+using ::testing::HasSubstr;
+
+constexpr char kConfig[] = R"(
+  calculator: "SceneCroppingCalculator"
+  input_stream: "VIDEO_FRAMES:camera_frames_org"
+  input_stream: "KEY_FRAMES:down_sampled_frames"
+  input_stream: "DETECTION_FEATURES:salient_regions"
+  input_stream: "STATIC_FEATURES:border_features"
+  input_stream: "SHOT_BOUNDARIES:shot_boundary_frames"
+  output_stream: "CROPPED_FRAMES:cropped_frames"
+  options: {
+    [mediapipe.autoflip.SceneCroppingCalculatorOptions.ext]: {
+      target_width: $0
+      target_height: $1
+      target_size_type: $2
+      max_scene_size: $3
+      prior_frame_buffer_size: $4
+    }
+  })";
+
+constexpr char kNoKeyFrameConfig[] = R"(
+  calculator: "SceneCroppingCalculator"
+  input_stream: "VIDEO_FRAMES:camera_frames_org"
+  input_stream: "DETECTION_FEATURES:salient_regions"
+  input_stream: "STATIC_FEATURES:border_features"
+  input_stream: "SHOT_BOUNDARIES:shot_boundary_frames"
+  output_stream: "CROPPED_FRAMES:cropped_frames"
+  options: {
+    [mediapipe.autoflip.SceneCroppingCalculatorOptions.ext]: {
+      target_width: $0
+      target_height: $1
+    }
+  })";
+
+constexpr char kDebugConfig[] = R"(
+  calculator: "SceneCroppingCalculator"
+  input_stream: "VIDEO_FRAMES:camera_frames_org"
+  input_stream: "KEY_FRAMES:down_sampled_frames"
+  input_stream: "DETECTION_FEATURES:salient_regions"
+  input_stream: "STATIC_FEATURES:border_features"
+  input_stream: "SHOT_BOUNDARIES:shot_boundary_frames"
+  output_stream: "CROPPED_FRAMES:cropped_frames"
+  output_stream: "KEY_FRAME_CROP_REGION_VIZ_FRAMES:key_frame_crop_viz_frames"
+  output_stream: "SALIENT_POINT_FRAME_VIZ_FRAMES:salient_point_viz_frames"
+  output_stream: "CROPPING_SUMMARY:cropping_summaries"
+  options: {
+    [mediapipe.autoflip.SceneCroppingCalculatorOptions.ext]: {
+      target_width: $0
+      target_height: $1
+    }
+  })";
+
+constexpr int kInputFrameWidth = 1280;
+constexpr int kInputFrameHeight = 720;
+
+constexpr int kKeyFrameWidth = 640;
+constexpr int kKeyFrameHeight = 360;
+
+constexpr int kTargetWidth = 720;
+constexpr int kTargetHeight = 1124;
+constexpr SceneCroppingCalculatorOptions::TargetSizeType kTargetSizeType =
+    SceneCroppingCalculatorOptions::USE_TARGET_DIMENSION;
+
+constexpr int kNumScenes = 3;
+constexpr int kSceneSize = 8;
+constexpr int kMaxSceneSize = 10;
+constexpr int kPriorFrameBufferSize = 5;
+
+constexpr int kMinNumDetections = 0;
+constexpr int kMaxNumDetections = 10;
+
+constexpr int kDownSampleRate = 4;
+constexpr int64 kTimestampDiff = 20000;
+
+// Returns a singleton random engine for generating random values. The seed is
+// fixed for reproducibility.
+std::default_random_engine& GetGen() {
+  static std::default_random_engine generator{0};
+  return generator;
+}
+
+// Returns random color with r, g, b in the range of [0, 255].
+cv::Scalar GetRandomColor() {
+  std::uniform_int_distribution<int> distribution(0, 255);
+  const int red = distribution(GetGen());
+  const int green = distribution(GetGen());
+  const int blue = distribution(GetGen());
+  return cv::Scalar(red, green, blue);
+}
+
+// Makes a detection set given number of detections. Each detection has randomly
+// generated regions within given width and height with random score in [0, 1],
+// and is randomly set to be required or non-required.
+std::unique_ptr<DetectionSet> MakeDetections(const int num_detections,
+                                             const int width,
+                                             const int height) {
+  std::uniform_int_distribution<int> width_distribution(0, width);
+  std::uniform_int_distribution<int> height_distribution(0, height);
+  std::uniform_real_distribution<float> score_distribution(0.0, 1.0);
+  std::bernoulli_distribution is_required_distribution(0.5);
+  auto detections = absl::make_unique<DetectionSet>();
+  for (int i = 0; i < num_detections; ++i) {
+    auto* region = detections->add_detections();
+    const int x1 = width_distribution(GetGen());
+    const int x2 = width_distribution(GetGen());
+    const int y1 = height_distribution(GetGen());
+    const int y2 = height_distribution(GetGen());
+    const int x_min = std::min(x1, x2), x_max = std::max(x1, x2);
+    const int y_min = std::min(y1, y2), y_max = std::max(y1, y2);
+    auto* location = region->mutable_location();
+    location->set_x(x_min);
+    location->set_width(x_max - x_min);
+    location->set_y(y_min);
+    location->set_height(y_max - y_min);
+    region->set_score(score_distribution(GetGen()));
+    region->set_is_required(is_required_distribution(GetGen()));
+  }
+  return detections;
+}
+
+// Makes an image frame of solid color given color, width, and height.
+std::unique_ptr<ImageFrame> MakeImageFrameFromColor(const cv::Scalar& color,
+                                                    const int width,
+                                                    const int height) {
+  auto image_frame =
+      absl::make_unique<ImageFrame>(ImageFormat::SRGB, width, height);
+  auto mat = formats::MatView(image_frame.get());
+  mat = color;
+  return image_frame;
+}
+
+// Adds key frame detection features given time (in ms) to the input stream.
+// Randomly generates a number of detections in the range of kMinNumDetections
+// and kMaxNumDetections. Optionally add a key image frame of random solid color
+// and given size.
+void AddKeyFrameFeatures(const int64 time_ms, const int key_frame_width,
+                         const int key_frame_height,
+                         CalculatorRunner::StreamContentsSet* inputs) {
+  Timestamp timestamp(time_ms);
+  if (inputs->HasTag("KEY_FRAMES")) {
+    auto key_frame = MakeImageFrameFromColor(GetRandomColor(), key_frame_width,
+                                             key_frame_height);
+    inputs->Tag("KEY_FRAMES")
+        .packets.push_back(Adopt(key_frame.release()).At(timestamp));
+  }
+
+  const int num_detections = std::uniform_int_distribution<int>(
+      kMinNumDetections, kMaxNumDetections)(GetGen());
+  auto detections =
+      MakeDetections(num_detections, key_frame_width, key_frame_height);
+  inputs->Tag("DETECTION_FEATURES")
+      .packets.push_back(Adopt(detections.release()).At(timestamp));
+}
+
+// Adds a scene given number of frames to the input stream. Spaces frame at the
+// default timestamp interval starting from given start frame index. Scene has
+// empty static features.
+void AddScene(const int start_frame_index, const int num_scene_frames,
+              const int frame_width, const int frame_height,
+              const int key_frame_width, const int key_frame_height,
+              CalculatorRunner::StreamContentsSet* inputs) {
+  int64 time_ms = start_frame_index * kTimestampDiff;
+  for (int i = 0; i < num_scene_frames; ++i) {
+    Timestamp timestamp(time_ms);
+    auto frame =
+        MakeImageFrameFromColor(GetRandomColor(), frame_width, frame_height);
+    inputs->Tag("VIDEO_FRAMES")
+        .packets.push_back(Adopt(frame.release()).At(timestamp));
+    auto static_features = absl::make_unique<StaticFeatures>();
+    inputs->Tag("STATIC_FEATURES")
+        .packets.push_back(Adopt(static_features.release()).At(timestamp));
+    if (i % kDownSampleRate == 0) {  // is a key frame
+      AddKeyFrameFeatures(time_ms, key_frame_width, key_frame_height, inputs);
+    }
+    if (i == num_scene_frames - 1) {  // adds shot boundary
+      inputs->Tag("SHOT_BOUNDARIES")
+          .packets.push_back(Adopt(new bool(true)).At(Timestamp(time_ms)));
+    }
+    time_ms += kTimestampDiff;
+  }
+}
+
+// Checks that the output stream for cropped frames has the correct number of
+// frames, and that the size of each frame is correct.
+void CheckCroppedFrames(const CalculatorRunner& runner, const int num_frames,
+                        const int target_width, const int target_height) {
+  const auto& outputs = runner.Outputs();
+  EXPECT_TRUE(outputs.HasTag("CROPPED_FRAMES"));
+  const auto& cropped_frames_outputs = outputs.Tag("CROPPED_FRAMES").packets;
+  EXPECT_EQ(cropped_frames_outputs.size(), num_frames);
+  for (int i = 0; i < num_frames; ++i) {
+    const auto& cropped_frame = cropped_frames_outputs[i].Get<ImageFrame>();
+    EXPECT_EQ(cropped_frame.Width(), target_width);
+    EXPECT_EQ(cropped_frame.Height(), target_height);
+  }
+}
+
+// Checks that the calculator checks the maximum scene size is valid.
+TEST(SceneCroppingCalculatorTest, ChecksMaxSceneSize) {
+  const CalculatorGraphConfig::Node config =
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(
+          absl::Substitute(kConfig, kTargetWidth, kTargetHeight,
+                           kTargetSizeType, 0, kPriorFrameBufferSize));
+  auto runner = absl::make_unique<CalculatorRunner>(config);
+  const auto status = runner->Run();
+  EXPECT_FALSE(status.ok());
+  EXPECT_THAT(status.ToString(),
+              HasSubstr("Maximum scene size is non-positive."));
+}
+
+// Checks that the calculator checks the prior frame buffer size is valid.
+TEST(SceneCroppingCalculatorTest, ChecksPriorFrameBufferSize) {
+  const CalculatorGraphConfig::Node config =
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(
+          absl::Substitute(kConfig, kTargetWidth, kTargetHeight,
+                           kTargetSizeType, kMaxSceneSize, -1));
+  auto runner = absl::make_unique<CalculatorRunner>(config);
+  const auto status = runner->Run();
+  EXPECT_FALSE(status.ok());
+  EXPECT_THAT(status.ToString(),
+              HasSubstr("Prior frame buffer size is negative."));
+}
+
+// Checks that the calculator crops scene frames when there is no input key
+// frames stream.
+TEST(SceneCroppingCalculatorTest, HandlesNoKeyFrames) {
+  const CalculatorGraphConfig::Node config =
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(
+          absl::Substitute(kNoKeyFrameConfig, kTargetWidth, kTargetHeight));
+  auto runner = absl::make_unique<CalculatorRunner>(config);
+  AddScene(0, kSceneSize, kInputFrameWidth, kInputFrameHeight, kKeyFrameWidth,
+           kKeyFrameHeight, runner->MutableInputs());
+  MP_EXPECT_OK(runner->Run());
+  CheckCroppedFrames(*runner, kSceneSize, kTargetWidth, kTargetHeight);
+}
+
+// Checks that the calculator handles scenes longer than maximum scene size (
+// force flush is triggered).
+TEST(SceneCroppingCalculatorTest, HandlesLongScene) {
+  const CalculatorGraphConfig::Node config =
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(absl::Substitute(
+          kConfig, kTargetWidth, kTargetHeight, kTargetSizeType, kMaxSceneSize,
+          kPriorFrameBufferSize));
+  auto runner = absl::make_unique<CalculatorRunner>(config);
+  AddScene(0, 2 * kMaxSceneSize, kInputFrameWidth, kInputFrameHeight,
+           kKeyFrameWidth, kKeyFrameHeight, runner->MutableInputs());
+  MP_EXPECT_OK(runner->Run());
+  CheckCroppedFrames(*runner, 2 * kMaxSceneSize, kTargetWidth, kTargetHeight);
+}
+
+// Checks that the calculator can optionally output debug streams.
+TEST(SceneCroppingCalculatorTest, OutputsDebugStreams) {
+  const CalculatorGraphConfig::Node config =
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(
+          absl::Substitute(kDebugConfig, kTargetWidth, kTargetHeight));
+  auto runner = absl::make_unique<CalculatorRunner>(config);
+  const int num_frames = kSceneSize;
+  AddScene(0, num_frames, kInputFrameWidth, kInputFrameHeight, kKeyFrameWidth,
+           kKeyFrameHeight, runner->MutableInputs());
+
+  MP_EXPECT_OK(runner->Run());
+  const auto& outputs = runner->Outputs();
+  EXPECT_TRUE(outputs.HasTag("KEY_FRAME_CROP_REGION_VIZ_FRAMES"));
+  EXPECT_TRUE(outputs.HasTag("SALIENT_POINT_FRAME_VIZ_FRAMES"));
+  EXPECT_TRUE(outputs.HasTag("CROPPING_SUMMARY"));
+  const auto& crop_region_viz_frames_outputs =
+      outputs.Tag("KEY_FRAME_CROP_REGION_VIZ_FRAMES").packets;
+  const auto& salient_point_viz_frames_outputs =
+      outputs.Tag("SALIENT_POINT_FRAME_VIZ_FRAMES").packets;
+  const auto& summary_output = outputs.Tag("CROPPING_SUMMARY").packets;
+  EXPECT_EQ(crop_region_viz_frames_outputs.size(), num_frames);
+  EXPECT_EQ(salient_point_viz_frames_outputs.size(), num_frames);
+  EXPECT_EQ(summary_output.size(), 1);
+
+  for (int i = 0; i < num_frames; ++i) {
+    const auto& crop_region_viz_frame =
+        crop_region_viz_frames_outputs[i].Get<ImageFrame>();
+    EXPECT_EQ(crop_region_viz_frame.Width(), kInputFrameWidth);
+    EXPECT_EQ(crop_region_viz_frame.Height(), kInputFrameHeight);
+    const auto& salient_point_viz_frame =
+        salient_point_viz_frames_outputs[i].Get<ImageFrame>();
+    EXPECT_EQ(salient_point_viz_frame.Width(), kInputFrameWidth);
+    EXPECT_EQ(salient_point_viz_frame.Height(), kInputFrameHeight);
+  }
+  const auto& summary = summary_output[0].Get<VideoCroppingSummary>();
+  EXPECT_EQ(summary.scene_summaries_size(), 2);
+  const auto& summary_0 = summary.scene_summaries(0);
+  EXPECT_TRUE(summary_0.is_padded());
+  EXPECT_TRUE(summary_0.camera_motion().has_steady_motion());
+}
+
+// Checks that the calculator handles the case of generating landscape frames.
+TEST(SceneCroppingCalculatorTest, HandlesLandscapeTarget) {
+  const int input_width = 900;
+  const int input_height = 1600;
+  const int target_width = 1200;
+  const int target_height = 800;
+  const CalculatorGraphConfig::Node config =
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(absl::Substitute(
+          kConfig, target_width, target_height, kTargetSizeType, kMaxSceneSize,
+          kPriorFrameBufferSize));
+  auto runner = absl::make_unique<CalculatorRunner>(config);
+  for (int i = 0; i < kNumScenes; ++i) {
+    AddScene(i * kSceneSize, kSceneSize, input_width, input_height,
+             kKeyFrameWidth, kKeyFrameHeight, runner->MutableInputs());
+  }
+  const int num_frames = kSceneSize * kNumScenes;
+  MP_EXPECT_OK(runner->Run());
+  CheckCroppedFrames(*runner, num_frames, target_width, target_height);
+}
+
+// Checks that the calculator crops scene frames to target size when the target
+// size type is the default USE_TARGET_DIMENSION.
+TEST(SceneCroppingCalculatorTest, CropsToTargetSize) {
+  const CalculatorGraphConfig::Node config =
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(absl::Substitute(
+          kConfig, kTargetWidth, kTargetHeight, kTargetSizeType, kMaxSceneSize,
+          kPriorFrameBufferSize));
+  auto runner = absl::make_unique<CalculatorRunner>(config);
+  for (int i = 0; i < kNumScenes; ++i) {
+    AddScene(i * kSceneSize, kSceneSize, kInputFrameWidth, kInputFrameHeight,
+             kKeyFrameWidth, kKeyFrameHeight, runner->MutableInputs());
+  }
+  const int num_frames = kSceneSize * kNumScenes;
+  MP_EXPECT_OK(runner->Run());
+  CheckCroppedFrames(*runner, num_frames, kTargetWidth, kTargetHeight);
+}
+
+// Checks that the calculator keeps original height if the target size type is
+// set to KEEP_ORIGINAL_HEIGHT.
+TEST(SceneCroppingCalculatorTest, KeepsOriginalHeight) {
+  const auto target_size_type =
+      SceneCroppingCalculatorOptions::KEEP_ORIGINAL_HEIGHT;
+  const int target_height = kInputFrameHeight;
+  const double target_aspect_ratio =
+      static_cast<double>(kTargetWidth) / kTargetHeight;
+  int target_width = std::round(target_height * target_aspect_ratio);
+  if (target_width % 2 == 1) target_width--;
+  const CalculatorGraphConfig::Node config =
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(absl::Substitute(
+          kConfig, kTargetWidth, kTargetHeight, target_size_type, kMaxSceneSize,
+          kPriorFrameBufferSize));
+  auto runner = absl::make_unique<CalculatorRunner>(config);
+  AddScene(0, kMaxSceneSize, kInputFrameWidth, kInputFrameHeight,
+           kKeyFrameWidth, kKeyFrameHeight, runner->MutableInputs());
+  MP_EXPECT_OK(runner->Run());
+  CheckCroppedFrames(*runner, kMaxSceneSize, target_width, target_height);
+}
+
+// Checks that the calculator keeps original width if the target size type is
+// set to KEEP_ORIGINAL_WIDTH.
+TEST(SceneCroppingCalculatorTest, KeepsOriginalWidth) {
+  const auto target_size_type =
+      SceneCroppingCalculatorOptions::KEEP_ORIGINAL_WIDTH;
+  const int target_width = kInputFrameWidth;
+  const double target_aspect_ratio =
+      static_cast<double>(kTargetWidth) / kTargetHeight;
+  int target_height = std::round(target_width / target_aspect_ratio);
+  if (target_height % 2 == 1) target_height--;
+  const CalculatorGraphConfig::Node config =
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(absl::Substitute(
+          kConfig, kTargetWidth, kTargetHeight, target_size_type, kMaxSceneSize,
+          kPriorFrameBufferSize));
+  auto runner = absl::make_unique<CalculatorRunner>(config);
+  AddScene(0, kMaxSceneSize, kInputFrameWidth, kInputFrameHeight,
+           kKeyFrameWidth, kKeyFrameHeight, runner->MutableInputs());
+  MP_EXPECT_OK(runner->Run());
+  CheckCroppedFrames(*runner, kMaxSceneSize, target_width, target_height);
+}
+
+// Checks that the calculator rejects odd target size.
+TEST(SceneCroppingCalculatorTest, RejectsOddTargetSize) {
+  const CalculatorGraphConfig::Node config =
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(absl::Substitute(
+          kConfig, kTargetWidth - 1, kTargetHeight, kTargetSizeType,
+          kMaxSceneSize, kPriorFrameBufferSize));
+  auto runner = absl::make_unique<CalculatorRunner>(config);
+  AddScene(0, kMaxSceneSize, kInputFrameWidth, kInputFrameHeight,
+           kKeyFrameWidth, kKeyFrameHeight, runner->MutableInputs());
+  const auto status = runner->Run();
+  EXPECT_FALSE(status.ok());
+  EXPECT_THAT(status.ToString(), HasSubstr("Target width cannot be odd"));
+}
+
+// Checks that the calculator always produces even frame size given even input
+// frame size and even target under all target size types.
+TEST(SceneCroppingCalculatorTest, ProducesEvenFrameSize) {
+  // Some commonly used video resolution (some are divided by 10 to make the
+  // test faster), and some odd input frame sizes.
+  const std::vector<std::pair<int, int>> video_sizes = {
+      {384, 216}, {256, 144}, {192, 108}, {128, 72},  {640, 360},
+      {426, 240}, {100, 100}, {214, 100}, {240, 100}, {720, 1124},
+      {90, 160},  {641, 360}, {640, 361}, {101, 101}};
+
+  const std::vector<SceneCroppingCalculatorOptions::TargetSizeType>
+      target_size_types = {SceneCroppingCalculatorOptions::USE_TARGET_DIMENSION,
+                           SceneCroppingCalculatorOptions::KEEP_ORIGINAL_HEIGHT,
+                           SceneCroppingCalculatorOptions::KEEP_ORIGINAL_WIDTH};
+
+  // Exhaustive check on each size as input and each size as output for each
+  // target size type.
+  for (int i = 0; i < video_sizes.size(); ++i) {
+    const int frame_width = video_sizes[i].first;
+    const int frame_height = video_sizes[i].second;
+    for (int j = 0; j < video_sizes.size(); ++j) {
+      const int target_width = video_sizes[j].first;
+      const int target_height = video_sizes[j].second;
+      if (target_width % 2 == 1 || target_height % 2 == 1) continue;
+      for (int k = 0; k < target_size_types.size(); ++k) {
+        const CalculatorGraphConfig::Node config =
+            ParseTextProtoOrDie<CalculatorGraphConfig::Node>(absl::Substitute(
+                kConfig, target_width, target_height, target_size_types[k],
+                kMaxSceneSize, kPriorFrameBufferSize));
+        auto runner = absl::make_unique<CalculatorRunner>(config);
+        AddScene(0, 1, frame_width, frame_height, kKeyFrameWidth,
+                 kKeyFrameHeight, runner->MutableInputs());
+        MP_EXPECT_OK(runner->Run());
+        const auto& output_frame = runner->Outputs()
+                                       .Tag("CROPPED_FRAMES")
+                                       .packets[0]
+                                       .Get<ImageFrame>();
+        EXPECT_EQ(output_frame.Width() % 2, 0);
+        EXPECT_EQ(output_frame.Height() % 2, 0);
+        if (target_size_types[k] ==
+            SceneCroppingCalculatorOptions::USE_TARGET_DIMENSION) {
+          EXPECT_EQ(output_frame.Width(), target_width);
+          EXPECT_EQ(output_frame.Height(), target_height);
+        } else if (target_size_types[k] ==
+                   SceneCroppingCalculatorOptions::KEEP_ORIGINAL_HEIGHT) {
+          // Difference could be 1 if input size is odd.
+          EXPECT_LE(std::abs(output_frame.Height() - frame_height), 1);
+        } else if (target_size_types[k] ==
+                   SceneCroppingCalculatorOptions::KEEP_ORIGINAL_WIDTH) {
+          EXPECT_LE(std::abs(output_frame.Width() - frame_width), 1);
+        }
+      }
+    }
+  }
+}
+
+// Checks that the calculator pads the frames with solid color when possible.
+TEST(SceneCroppingCalculatorTest, PadsWithSolidColorFromStaticFeatures) {
+  const int target_width = 100, target_height = 200;
+  const int input_width = 100, input_height = 100;
+  CalculatorGraphConfig::Node config =
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(
+          absl::Substitute(kNoKeyFrameConfig, target_width, target_height));
+  auto* options = config.mutable_options()->MutableExtension(
+      SceneCroppingCalculatorOptions::ext);
+  options->set_solid_background_frames_padding_fraction(0.6);
+  auto runner = absl::make_unique<CalculatorRunner>(config);
+
+  const int static_features_downsample_rate = 2;
+  const float fraction_with_solid_background = 0.7;
+  const int red = 122, green = 167, blue = 250;
+  const int num_frames_with_solid_background =
+      std::round(fraction_with_solid_background * kSceneSize /
+                 static_features_downsample_rate);
+
+  // Add inputs.
+  auto* inputs = runner->MutableInputs();
+  int64 time_ms = 0;
+  int num_static_features = 0;
+  for (int i = 0; i < kSceneSize; ++i) {
+    Timestamp timestamp(time_ms);
+    auto frame =
+        MakeImageFrameFromColor(GetRandomColor(), input_width, input_height);
+    inputs->Tag("VIDEO_FRAMES")
+        .packets.push_back(Adopt(frame.release()).At(timestamp));
+    if (i % static_features_downsample_rate == 0) {
+      auto static_features = absl::make_unique<StaticFeatures>();
+      if (num_static_features < num_frames_with_solid_background) {
+        auto* color = static_features->mutable_solid_background();
+        // Uses BGR to mimic input from static features solid background color.
+        color->set_r(blue);
+        color->set_g(green);
+        color->set_b(red);
+      }
+      inputs->Tag("STATIC_FEATURES")
+          .packets.push_back(Adopt(static_features.release()).At(timestamp));
+      num_static_features++;
+    }
+    if (i % kDownSampleRate == 0) {  // is a key frame
+      // Target crop size is (50, 100). Adds one required detection with size
+      // (80, 100) larger than the target crop size to force padding.
+      auto detections = absl::make_unique<DetectionSet>();
+      auto* salient_region = detections->add_detections();
+      salient_region->set_is_required(true);
+      auto* location = salient_region->mutable_location();
+      location->set_x(10);
+      location->set_y(0);
+      location->set_width(80);
+      location->set_height(input_height);
+      inputs->Tag("DETECTION_FEATURES")
+          .packets.push_back(Adopt(detections.release()).At(timestamp));
+    }
+    time_ms += kTimestampDiff;
+  }
+
+  MP_EXPECT_OK(runner->Run());
+
+  // Checks that the top and bottom borders indeed have the background color.
+  const int border_size = 37;
+  const auto& cropped_frames_outputs =
+      runner->Outputs().Tag("CROPPED_FRAMES").packets;
+  EXPECT_EQ(cropped_frames_outputs.size(), kSceneSize);
+  for (int i = 0; i < kSceneSize; ++i) {
+    const auto& cropped_frame = cropped_frames_outputs[i].Get<ImageFrame>();
+    cv::Mat mat = formats::MatView(&cropped_frame);
+    for (int x = 0; x < target_width; ++x) {
+      for (int y = 0; y < border_size; ++y) {
+        EXPECT_EQ(mat.at<cv::Vec3b>(y, x)[0], red);
+        EXPECT_EQ(mat.at<cv::Vec3b>(y, x)[1], green);
+        EXPECT_EQ(mat.at<cv::Vec3b>(y, x)[2], blue);
+      }
+      for (int y2 = 0; y2 < border_size; ++y2) {
+        const int y = target_height - 1 - y2;
+        EXPECT_EQ(mat.at<cv::Vec3b>(y, x)[0], red);
+        EXPECT_EQ(mat.at<cv::Vec3b>(y, x)[1], green);
+        EXPECT_EQ(mat.at<cv::Vec3b>(y, x)[2], blue);
+      }
+    }
+  }
+}
+
+// Checks that the calculator removes static borders from frames.
+TEST(SceneCroppingCalculatorTest, RemovesStaticBorders) {
+  const int target_width = 50, target_height = 100;
+  const int input_width = 100, input_height = 100;
+  const int top_border_size = 20, bottom_border_size = 20;
+  const cv::Rect top_border_rect(0, 0, input_width, top_border_size);
+  const cv::Rect bottom_border_rect(0, input_height - bottom_border_size,
+                                    input_width, bottom_border_size);
+  const cv::Scalar frame_color = cv::Scalar(255, 255, 255);
+  const cv::Scalar border_color = cv::Scalar(0, 0, 0);
+
+  const auto config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(
+      absl::Substitute(kNoKeyFrameConfig, target_width, target_height));
+  auto runner = absl::make_unique<CalculatorRunner>(config);
+
+  // Add inputs.
+  auto* inputs = runner->MutableInputs();
+  const auto timestamp = Timestamp(0);
+  // Make frame with borders.
+  auto frame = MakeImageFrameFromColor(frame_color, input_width, input_height);
+  auto mat = formats::MatView(frame.get());
+  mat(top_border_rect) = border_color;
+  mat(bottom_border_rect) = border_color;
+  inputs->Tag("VIDEO_FRAMES")
+      .packets.push_back(Adopt(frame.release()).At(timestamp));
+  // Set borders in static features.
+  auto static_features = absl::make_unique<StaticFeatures>();
+  auto* top_part = static_features->add_border();
+  top_part->set_relative_position(Border::TOP);
+  top_part->mutable_border_position()->set_height(top_border_size);
+  auto* bottom_part = static_features->add_border();
+  bottom_part->set_relative_position(Border::BOTTOM);
+  bottom_part->mutable_border_position()->set_height(bottom_border_size);
+  inputs->Tag("STATIC_FEATURES")
+      .packets.push_back(Adopt(static_features.release()).At(timestamp));
+  // Add empty detections to ensure no padding is used.
+  auto detections = absl::make_unique<DetectionSet>();
+  inputs->Tag("DETECTION_FEATURES")
+      .packets.push_back(Adopt(detections.release()).At(timestamp));
+
+  MP_EXPECT_OK(runner->Run());
+
+  // Checks that the top and bottom borders are removed. Each frame should have
+  // solid color equal to frame color.
+  const auto& cropped_frames_outputs =
+      runner->Outputs().Tag("CROPPED_FRAMES").packets;
+  EXPECT_EQ(cropped_frames_outputs.size(), 1);
+  const auto& cropped_frame = cropped_frames_outputs[0].Get<ImageFrame>();
+  const auto cropped_mat = formats::MatView(&cropped_frame);
+  for (int x = 0; x < target_width; ++x) {
+    for (int y = 0; y < target_height; ++y) {
+      EXPECT_EQ(cropped_mat.at<cv::Vec3b>(y, x)[0], frame_color[0]);
+      EXPECT_EQ(cropped_mat.at<cv::Vec3b>(y, x)[1], frame_color[1]);
+      EXPECT_EQ(cropped_mat.at<cv::Vec3b>(y, x)[2], frame_color[2]);
+    }
+  }
+}
+}  // namespace
+}  // namespace autoflip
+}  // namespace mediapipe

mediapipe/examples/desktop/autoflip/calculators/shot_boundary_calculator.cc

@@ -0,0 +1,190 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <map>
+#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "mediapipe/examples/desktop/autoflip/calculators/shot_boundary_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/image_frame.h"
+#include "mediapipe/framework/formats/image_frame_opencv.h"
+#include "mediapipe/framework/port/opencv_imgproc_inc.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/timestamp.h"
+
+using mediapipe::ImageFrame;
+using mediapipe::PacketTypeSet;
+
+// IO labels.
+constexpr char kVideoInputTag[] = "VIDEO";
+constexpr char kShotChangeTag[] = "IS_SHOT_CHANGE";
+// Histogram settings.
+const int kSaturationBins = 8;
+const int kHistogramChannels[] = {0, 1, 2};
+const int kHistogramBinNum[] = {kSaturationBins, kSaturationBins,
+                                kSaturationBins};
+const float kRange[] = {0, 256};
+const float* kHistogramRange[] = {kRange, kRange, kRange};
+
+namespace mediapipe {
+namespace autoflip {
+
+// This calculator computes a shot (or scene) change within a video.  It works
+// by computing a 3d color histogram and comparing this frame-to-frame. Settings
+// to control the shot change logic are presented in the options proto.
+//
+// Example:
+//  node {
+//    calculator: "ShotBoundaryCalculator"
+//    input_stream: "VIDEO:camera_frames"
+//    output_stream: "IS_SHOT_CHANGE:is_shot"
+//  }
+class ShotBoundaryCalculator : public mediapipe::CalculatorBase {
+ public:
+  ShotBoundaryCalculator() {}
+  ShotBoundaryCalculator(const ShotBoundaryCalculator&) = delete;
+  ShotBoundaryCalculator& operator=(const ShotBoundaryCalculator&) = delete;
+
+  static ::mediapipe::Status GetContract(mediapipe::CalculatorContract* cc);
+  mediapipe::Status Open(mediapipe::CalculatorContext* cc) override;
+  mediapipe::Status Process(mediapipe::CalculatorContext* cc) override;
+
+ private:
+  // Computes the histogram of an image.
+  void ComputeHistogram(const cv::Mat& image, cv::Mat* image_histogram);
+  // Transmits signal to next calculator.
+  void Transmit(mediapipe::CalculatorContext* cc, bool is_shot_change);
+  // Calculator options.
+  ShotBoundaryCalculatorOptions options_;
+  // Last time a shot was detected.
+  Timestamp last_shot_timestamp_;
+  // Defines if the calculator has received a frame yet.
+  bool init_;
+  // Histogram from the last frame.
+  cv::Mat last_histogram_;
+  // History of histogram motion.
+  std::deque<double> motion_history_;
+};
+REGISTER_CALCULATOR(ShotBoundaryCalculator);
+
+void ShotBoundaryCalculator::ComputeHistogram(const cv::Mat& image,
+                                              cv::Mat* image_histogram) {
+  cv::Mat equalized_image;
+  cv::cvtColor(image.clone(), equalized_image, CV_RGB2GRAY);
+
+  double min, max;
+  cv::minMaxLoc(equalized_image, &min, &max);
+
+  if (options_.equalize_histogram()) {
+    cv::equalizeHist(equalized_image, equalized_image);
+  }
+
+  cv::calcHist(&image, 1, kHistogramChannels, cv::Mat(), *image_histogram, 2,
+               kHistogramBinNum, kHistogramRange, true, false);
+}
+
+mediapipe::Status ShotBoundaryCalculator::Open(
+    mediapipe::CalculatorContext* cc) {
+  options_ = cc->Options<ShotBoundaryCalculatorOptions>();
+  last_shot_timestamp_ = Timestamp(0);
+  init_ = false;
+  return ::mediapipe::OkStatus();
+}
+
+void ShotBoundaryCalculator::Transmit(mediapipe::CalculatorContext* cc,
+                                      bool is_shot_change) {
+  if ((cc->InputTimestamp() - last_shot_timestamp_).Seconds() <
+      options_.min_shot_span()) {
+    is_shot_change = false;
+  }
+  if (is_shot_change) {
+    LOG(INFO) << "Shot change at: " << cc->InputTimestamp().Seconds()
+              << " seconds.";
+    cc->Outputs()
+        .Tag(kShotChangeTag)
+        .AddPacket(Adopt(std::make_unique<bool>(true).release())
+                       .At(cc->InputTimestamp()));
+  } else if (!options_.output_only_on_change()) {
+    cc->Outputs()
+        .Tag(kShotChangeTag)
+        .AddPacket(Adopt(std::make_unique<bool>(false).release())
+                       .At(cc->InputTimestamp()));
+  }
+}
+
+::mediapipe::Status ShotBoundaryCalculator::Process(
+    mediapipe::CalculatorContext* cc) {
+  // Connect to input frame and make a mutable copy.
+  cv::Mat frame_org = mediapipe::formats::MatView(
+      &cc->Inputs().Tag(kVideoInputTag).Get<ImageFrame>());
+  cv::Mat frame = frame_org.clone();
+
+  // Extract histogram from the current frame.
+  cv::Mat current_histogram;
+  ComputeHistogram(frame, &current_histogram);
+
+  if (!init_) {
+    last_histogram_ = current_histogram;
+    init_ = true;
+    Transmit(cc, false);
+    return ::mediapipe::OkStatus();
+  }
+
+  double current_motion_estimate =
+      1 - cv::compareHist(current_histogram, last_histogram_, CV_COMP_CORREL);
+  last_histogram_ = current_histogram;
+  motion_history_.push_front(current_motion_estimate);
+
+  if (motion_history_.size() != options_.window_size()) {
+    Transmit(cc, false);
+    return ::mediapipe::OkStatus();
+  }
+
+  // Shot detection algorithm is a mixture of adaptive (controlled with
+  // shot_measure) and hard thresholds. In saturation it uses hard thresholds
+  // to account for black startups, shot cuts across high motion etc.
+  // In the operating region it uses an adaptive threshold to tune motion vs.
+  // cut boundary.
+  double current_max =
+      *std::max_element(motion_history_.begin(), motion_history_.end());
+  double shot_measure = current_motion_estimate / current_max;
+
+  if ((shot_measure > options_.min_shot_measure() &&
+       current_motion_estimate > options_.min_motion_with_shot_measure()) ||
+      current_motion_estimate > options_.min_motion()) {
+    Transmit(cc, true);
+    last_shot_timestamp_ = cc->InputTimestamp();
+  } else {
+    Transmit(cc, false);
+  }
+
+  // Store histogram for next frame.
+  last_histogram_ = current_histogram;
+  motion_history_.pop_back();
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status ShotBoundaryCalculator::GetContract(
+    mediapipe::CalculatorContract* cc) {
+  cc->Inputs().Tag(kVideoInputTag).Set<ImageFrame>();
+  cc->Outputs().Tag(kShotChangeTag).Set<bool>();
+  return ::mediapipe::OkStatus();
+}
+
+}  // namespace autoflip
+}  // namespace mediapipe

mediapipe/examples/desktop/autoflip/calculators/shot_boundary_calculator.proto

@@ -0,0 +1,48 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe.autoflip;
+
+import "mediapipe/framework/calculator.proto";
+
+message ShotBoundaryCalculatorOptions {
+  extend mediapipe.CalculatorOptions {
+    optional ShotBoundaryCalculatorOptions ext = 281194049;
+  }
+  // Parameters to shot detection algorithm. All the constraints (the fields
+  // named with 'min_') need to be satisfied for a frame to be a shot boundary.
+  //
+  // Minimum motion to be considered as a shot boundary frame.
+  optional double min_motion = 1 [default = 0.2];
+  // Minimum number of shot duration (in seconds).
+  optional double min_shot_span = 2 [default = 2];
+  // A window for computing shot measure (see the definition in min_shot_measure
+  // field).
+  optional int32 window_size = 3 [default = 7];
+  // Minimum shot measure to be considered as a shot boundary frame.
+  // Must also satisfy the min_motion_with_shot_measure constraint.
+  // The shot measure is defined as the ratio of the motion of the
+  // current frame to the maximum motion of the frames in the window (defined
+  // as window_size).
+  optional double min_shot_measure = 4 [default = 10];
+  // Minimum motion to be considered as a shot boundary frame.
+  // Must also satisfy the min_shot_measure constraint.
+  optional double min_motion_with_shot_measure = 5 [default = 0.05];
+  // Only send results if the shot value is true.
+  optional bool output_only_on_change = 6 [default = true];
+  // Perform histogram equalization before computing keypoints/features.
+  optional bool equalize_histogram = 7 [default = false];
+}

mediapipe/examples/desktop/autoflip/calculators/shot_boundary_calculator_test.cc

@@ -0,0 +1,163 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "absl/strings/string_view.h"
+#include "mediapipe/examples/desktop/autoflip/calculators/shot_boundary_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/deps/file_path.h"
+#include "mediapipe/framework/formats/image_frame.h"
+#include "mediapipe/framework/formats/image_frame_opencv.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/opencv_core_inc.h"
+#include "mediapipe/framework/port/opencv_imgcodecs_inc.h"
+#include "mediapipe/framework/port/opencv_imgproc_inc.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/port/status_matchers.h"
+
+using mediapipe::Adopt;
+using mediapipe::CalculatorGraphConfig;
+using mediapipe::CalculatorRunner;
+using mediapipe::ImageFormat;
+using mediapipe::ImageFrame;
+using mediapipe::PacketTypeSet;
+using mediapipe::ParseTextProtoOrDie;
+using mediapipe::Timestamp;
+
+namespace mediapipe {
+namespace autoflip {
+namespace {
+
+const char kConfig[] = R"(
+    calculator: "ShotBoundaryCalculator"
+    input_stream: "VIDEO:camera_frames"
+    output_stream: "IS_SHOT_CHANGE:is_shot"
+    )";
+const int kTestFrameWidth = 640;
+const int kTestFrameHeight = 480;
+
+void AddFrames(const int number_of_frames, const std::set<int>& skip_frames,
+               CalculatorRunner* runner) {
+  cv::Mat image =
+      cv::imread(file::JoinPath("./",
+                                "/mediapipe/examples/desktop/"
+                                "autoflip/calculators/testdata/dino.jpg"));
+
+  for (int i = 0; i < number_of_frames; i++) {
+    auto input_frame = ::absl::make_unique<ImageFrame>(
+        ImageFormat::SRGB, kTestFrameWidth, kTestFrameHeight);
+    cv::Mat input_mat = mediapipe::formats::MatView(input_frame.get());
+    input_mat.setTo(cv::Scalar(0, 0, 0));
+    cv::Mat sub_image =
+        image(cv::Rect(i, i, kTestFrameWidth, kTestFrameHeight));
+    cv::Mat frame_area =
+        input_mat(cv::Rect(0, 0, sub_image.cols, sub_image.rows));
+    if (skip_frames.count(i) < 1) {
+      sub_image.copyTo(frame_area);
+    }
+    runner->MutableInputs()->Tag("VIDEO").packets.push_back(
+        Adopt(input_frame.release()).At(Timestamp(i * 1000000)));
+  }
+}
+
+void CheckOutput(const int number_of_frames, const std::set<int>& shot_frames,
+                 const std::vector<Packet>& output_packets) {
+  ASSERT_EQ(number_of_frames, output_packets.size());
+  for (int i = 0; i < number_of_frames; i++) {
+    if (shot_frames.count(i) < 1) {
+      EXPECT_FALSE(output_packets[i].Get<bool>());
+    } else {
+      EXPECT_TRUE(output_packets[i].Get<bool>());
+    }
+  }
+}
+
+TEST(ShotBoundaryCalculatorTest, NoShotChange) {
+  CalculatorGraphConfig::Node node =
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfig);
+  node.mutable_options()
+      ->MutableExtension(ShotBoundaryCalculatorOptions::ext)
+      ->set_output_only_on_change(false);
+  auto runner = ::absl::make_unique<CalculatorRunner>(node);
+
+  AddFrames(10, {}, runner.get());
+  MP_ASSERT_OK(runner->Run());
+  CheckOutput(10, {}, runner->Outputs().Tag("IS_SHOT_CHANGE").packets);
+}
+
+TEST(ShotBoundaryCalculatorTest, ShotChangeSingle) {
+  CalculatorGraphConfig::Node node =
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfig);
+  node.mutable_options()
+      ->MutableExtension(ShotBoundaryCalculatorOptions::ext)
+      ->set_output_only_on_change(false);
+  auto runner = ::absl::make_unique<CalculatorRunner>(node);
+
+  AddFrames(20, {10}, runner.get());
+  MP_ASSERT_OK(runner->Run());
+  CheckOutput(20, {10}, runner->Outputs().Tag("IS_SHOT_CHANGE").packets);
+}
+
+TEST(ShotBoundaryCalculatorTest, ShotChangeDouble) {
+  CalculatorGraphConfig::Node node =
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfig);
+  node.mutable_options()
+      ->MutableExtension(ShotBoundaryCalculatorOptions::ext)
+      ->set_output_only_on_change(false);
+  auto runner = ::absl::make_unique<CalculatorRunner>(node);
+
+  AddFrames(20, {14, 17}, runner.get());
+  MP_ASSERT_OK(runner->Run());
+  CheckOutput(20, {14, 17}, runner->Outputs().Tag("IS_SHOT_CHANGE").packets);
+}
+
+TEST(ShotBoundaryCalculatorTest, ShotChangeFiltered) {
+  CalculatorGraphConfig::Node node =
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfig);
+  node.mutable_options()
+      ->MutableExtension(ShotBoundaryCalculatorOptions::ext)
+      ->set_min_shot_span(5);
+  node.mutable_options()
+      ->MutableExtension(ShotBoundaryCalculatorOptions::ext)
+      ->set_output_only_on_change(false);
+
+  auto runner = ::absl::make_unique<CalculatorRunner>(node);
+
+  AddFrames(24, {16, 19}, runner.get());
+  MP_ASSERT_OK(runner->Run());
+  CheckOutput(24, {16}, runner->Outputs().Tag("IS_SHOT_CHANGE").packets);
+}
+
+TEST(ShotBoundaryCalculatorTest, ShotChangeSingleOnOnChange) {
+  CalculatorGraphConfig::Node node =
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfig);
+  node.mutable_options()
+      ->MutableExtension(ShotBoundaryCalculatorOptions::ext)
+      ->set_output_only_on_change(true);
+  auto runner = ::absl::make_unique<CalculatorRunner>(node);
+
+  AddFrames(20, {15}, runner.get());
+  MP_ASSERT_OK(runner->Run());
+  auto output_packets = runner->Outputs().Tag("IS_SHOT_CHANGE").packets;
+  ASSERT_EQ(output_packets.size(), 1);
+  ASSERT_EQ(output_packets[0].Get<bool>(), true);
+  ASSERT_EQ(output_packets[0].Timestamp().Value(), 15000000);
+}
+
+}  // namespace
+}  // namespace autoflip
+}  // namespace mediapipe

mediapipe/examples/desktop/autoflip/calculators/signal_fusing_calculator.cc

@@ -0,0 +1,231 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <map>
+#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "mediapipe/examples/desktop/autoflip/autoflip_messages.pb.h"
+#include "mediapipe/examples/desktop/autoflip/calculators/signal_fusing_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+
+using mediapipe::Packet;
+using mediapipe::PacketTypeSet;
+using mediapipe::autoflip::DetectionSet;
+using mediapipe::autoflip::SalientRegion;
+using mediapipe::autoflip::SignalType;
+
+namespace mediapipe {
+namespace autoflip {
+
+struct InputSignal {
+  SalientRegion signal;
+  int source;
+};
+
+struct Frame {
+  std::vector<InputSignal> input_detections;
+  mediapipe::Timestamp time;
+};
+
+// This calculator takes one scene change signal and an arbitrary number of
+// detection signals and outputs a single list of detections.  The scores for
+// the detections can be re-normalized using the options proto.  Additionally,
+// if a detection has a consistent tracking id during a scene the score for that
+// detection is averaged over the whole scene.
+//
+// Example:
+//  node {
+//    calculator: "SignalFusingCalculator"
+//    input_stream: "scene_change"
+//    input_stream: "detection_faces"
+//    input_stream: "detection_custom_text"
+//    output_stream: "salient_region"
+//    options:{
+//    [mediapipe.autoflip.SignalFusingCalculatorOptions.ext]:{
+//      signal_settings{
+//        type: {standard: FACE}
+//        min_score: 0.5
+//        max_score: 0.6
+//      }
+//      signal_settings{
+//        type: {custom: "custom_text"}
+//        min_score: 0.9
+//        max_score: 1.0
+//      }
+//    }
+//    }
+//  }
+class SignalFusingCalculator : public mediapipe::CalculatorBase {
+ public:
+  SignalFusingCalculator() {}
+  SignalFusingCalculator(const SignalFusingCalculator&) = delete;
+  SignalFusingCalculator& operator=(const SignalFusingCalculator&) = delete;
+
+  static ::mediapipe::Status GetContract(mediapipe::CalculatorContract* cc);
+  mediapipe::Status Open(mediapipe::CalculatorContext* cc) override;
+  mediapipe::Status Process(mediapipe::CalculatorContext* cc) override;
+  mediapipe::Status Close(mediapipe::CalculatorContext* cc) override;
+
+ private:
+  mediapipe::Status ProcessScene(mediapipe::CalculatorContext* cc);
+  SignalFusingCalculatorOptions options_;
+  std::map<std::string, SignalSettings> settings_by_type_;
+  std::vector<Frame> scene_frames_;
+};
+REGISTER_CALCULATOR(SignalFusingCalculator);
+
+namespace {
+std::string CreateSettingsKey(const SignalType& signal_type) {
+  if (signal_type.has_standard()) {
+    return "standard_" + std::to_string(signal_type.standard());
+  } else {
+    return "custom_" + signal_type.custom();
+  }
+}
+std::string CreateKey(const InputSignal& detection) {
+  std::string id_source = std::to_string(detection.source);
+  std::string id_signal = std::to_string(detection.signal.tracking_id());
+  std::string id = id_source + ":" + id_signal;
+  return id;
+}
+
+}  // namespace
+
+mediapipe::Status SignalFusingCalculator::Open(
+    mediapipe::CalculatorContext* cc) {
+  options_ = cc->Options<SignalFusingCalculatorOptions>();
+  for (const auto& setting : options_.signal_settings()) {
+    settings_by_type_[CreateSettingsKey(setting.type())] = setting;
+  }
+  return ::mediapipe::OkStatus();
+}
+
+mediapipe::Status SignalFusingCalculator::Close(
+    mediapipe::CalculatorContext* cc) {
+  if (!scene_frames_.empty()) {
+    MP_RETURN_IF_ERROR(ProcessScene(cc));
+    scene_frames_.clear();
+  }
+  return ::mediapipe::OkStatus();
+}
+
+mediapipe::Status SignalFusingCalculator::ProcessScene(
+    mediapipe::CalculatorContext* cc) {
+  std::map<std::string, int> detection_count;
+  std::map<std::string, float> multiframe_score;
+  // Create a unified score for all items with temporal ids.
+  for (const Frame& frame : scene_frames_) {
+    for (const auto& detection : frame.input_detections) {
+      if (detection.signal.has_tracking_id()) {
+        // Create key for each detector type
+        if (detection_count.find(CreateKey(detection)) ==
+            detection_count.end()) {
+          multiframe_score[CreateKey(detection)] = 0.0;
+          detection_count[CreateKey(detection)] = 0;
+        }
+        multiframe_score[CreateKey(detection)] += detection.signal.score();
+        detection_count[CreateKey(detection)]++;
+      }
+    }
+  }
+
+  // Average scores.
+  for (auto iterator = multiframe_score.begin();
+       iterator != multiframe_score.end(); iterator++) {
+    multiframe_score[iterator->first] =
+        iterator->second / detection_count[iterator->first];
+  }
+
+  // Process detections.
+  for (const Frame& frame : scene_frames_) {
+    std::unique_ptr<DetectionSet> processed_detections(new DetectionSet());
+    for (auto detection : frame.input_detections) {
+      float score = detection.signal.score();
+      if (detection.signal.has_tracking_id()) {
+        std::string id_source = std::to_string(detection.source);
+        std::string id_signal = std::to_string(detection.signal.tracking_id());
+        std::string id = id_source + ":" + id_signal;
+        score = multiframe_score[id];
+      }
+      // Normalize within range.
+      float min_value = 0.0;
+      float max_value = 1.0;
+
+      auto settings_it = settings_by_type_.find(
+          CreateSettingsKey(detection.signal.signal_type()));
+      if (settings_it != settings_by_type_.end()) {
+        min_value = settings_it->second.min_score();
+        max_value = settings_it->second.max_score();
+        detection.signal.set_is_required(settings_it->second.is_required());
+      }
+
+      float final_score = score * (max_value - min_value) + min_value;
+      detection.signal.set_score(final_score);
+      *processed_detections->add_detections() = detection.signal;
+    }
+    cc->Outputs().Index(0).Add(processed_detections.release(), frame.time);
+  }
+
+  return ::mediapipe::OkStatus();
+}
+
+mediapipe::Status SignalFusingCalculator::Process(
+    mediapipe::CalculatorContext* cc) {
+  bool is_boundary = false;
+  if (!cc->Inputs().Index(0).Value().IsEmpty()) {
+    is_boundary = cc->Inputs().Index(0).Get<bool>();
+  }
+
+  if (is_boundary || scene_frames_.size() > options_.max_scene_size()) {
+    MP_RETURN_IF_ERROR(ProcessScene(cc));
+    scene_frames_.clear();
+  }
+
+  Frame frame;
+  for (int i = 1; i < cc->Inputs().NumEntries(); ++i) {
+    const Packet& packet = cc->Inputs().Index(i).Value();
+    if (packet.IsEmpty()) {
+      continue;
+    }
+    const auto& detection_set = packet.Get<autoflip::DetectionSet>();
+    for (const auto& detection : detection_set.detections()) {
+      InputSignal input;
+      input.signal = detection;
+      input.source = i;
+      frame.input_detections.push_back(input);
+    }
+  }
+  frame.time = cc->InputTimestamp();
+  scene_frames_.push_back(frame);
+
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status SignalFusingCalculator::GetContract(
+    mediapipe::CalculatorContract* cc) {
+  cc->Inputs().Index(0).Set<bool>();
+  for (int i = 1; i < cc->Inputs().NumEntries(); ++i) {
+    cc->Inputs().Index(i).Set<autoflip::DetectionSet>();
+  }
+  cc->Outputs().Index(0).Set<autoflip::DetectionSet>();
+  return ::mediapipe::OkStatus();
+}
+
+}  // namespace autoflip
+}  // namespace mediapipe

mediapipe/examples/desktop/autoflip/calculators/signal_fusing_calculator.proto

@@ -0,0 +1,54 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe.autoflip;
+
+import "mediapipe/examples/desktop/autoflip/autoflip_messages.proto";
+import "mediapipe/framework/calculator.proto";
+
+// Next tag: 3
+message SignalFusingCalculatorOptions {
+  extend mediapipe.CalculatorOptions {
+    optional SignalFusingCalculatorOptions ext = 280092372;
+  }
+  // Setting related to each type of signal this calculator could process.
+  repeated SignalSettings signal_settings = 1;
+
+  // Force a flush of the frame buffer after this number of frames.
+  optional int32 max_scene_size = 2 [default = 600];
+}
+
+// Next tag: 5
+message SignalSettings {
+  // The type of signal these settings pertain to.
+  optional SignalType type = 1;
+
+  // Force a normalized incoming score to be re-normalized to within this range.
+  // (set values to min:0 and max:1 for no change in the incoming score)
+  // Values must be between 0-1, min must be less than max.
+  //
+  // Example of score adjustment:
+  // Incoming OCR score: .7
+  // Min OCR Score: .9
+  // Max OCR Score: 1.0
+  // --Result: .97
+  optional float min_score = 2 [default = 0];
+  optional float max_score = 3 [default = 1.0];
+
+  // Is this signal required within the output cropped video?  If it is it will
+  // be included or the video will be marked as failed to convert.
+  optional bool is_required = 4 [default = false];
+}

mediapipe/examples/desktop/autoflip/calculators/signal_fusing_calculator_test.cc

@@ -0,0 +1,438 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "absl/strings/string_view.h"
+#include "mediapipe/examples/desktop/autoflip/autoflip_messages.pb.h"
+#include "mediapipe/examples/desktop/autoflip/calculators/signal_fusing_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/formats/image_frame.h"
+#include "mediapipe/framework/formats/image_frame_opencv.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/port/status_matchers.h"
+
+using mediapipe::autoflip::DetectionSet;
+
+namespace mediapipe {
+namespace autoflip {
+namespace {
+
+const char kConfigA[] = R"(
+    calculator: "SignalFusingCalculator"
+    input_stream: "scene_change"
+    input_stream: "detection_set_a"
+    input_stream: "detection_set_b"
+    output_stream: "salient_region"
+    options:{
+    [mediapipe.autoflip.SignalFusingCalculatorOptions.ext]:{
+      signal_settings{
+        type: {standard: FACE_FULL}
+        min_score: 0.5
+        max_score: 0.6
+      }
+      signal_settings{
+        type: {standard: TEXT}
+        min_score: 0.9
+        max_score: 1.0
+      }
+    }
+    })";
+
+const char kConfigB[] = R"(
+    calculator: "SignalFusingCalculator"
+    input_stream: "scene_change"
+    input_stream: "detection_set_a"
+    input_stream: "detection_set_b"
+    input_stream: "detection_set_c"
+    output_stream: "salient_region"
+    options:{
+    [mediapipe.autoflip.SignalFusingCalculatorOptions.ext]:{
+      signal_settings{
+        type: {standard: FACE_FULL}
+        min_score: 0.5
+        max_score: 0.6
+      }
+      signal_settings{
+        type: {custom: "text"}
+        min_score: 0.9
+        max_score: 1.0
+      }
+      signal_settings{
+        type: {standard: LOGO}
+        min_score: 0.1
+        max_score: 0.3
+      }
+    }
+    })";
+
+TEST(SignalFusingCalculatorTest, TwoInputNoTracking) {
+  auto runner = absl::make_unique<CalculatorRunner>(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfigA));
+
+  auto input_border = absl::make_unique<bool>(false);
+  runner->MutableInputs()->Index(0).packets.push_back(
+      Adopt(input_border.release()).At(Timestamp(0)));
+
+  auto input_face =
+      absl::make_unique<DetectionSet>(ParseTextProtoOrDie<DetectionSet>(
+          R"(
+            detections {
+              score: 0.5
+              signal_type: { standard: FACE_FULL }
+            }
+            detections {
+              score: 0.3
+              signal_type: { standard: FACE_FULL }
+            }
+          )"));
+
+  runner->MutableInputs()->Index(1).packets.push_back(
+      Adopt(input_face.release()).At(Timestamp(0)));
+
+  auto input_ocr =
+      absl::make_unique<DetectionSet>(ParseTextProtoOrDie<DetectionSet>(
+          R"(
+            detections {
+              score: 0.3
+              signal_type: { standard: TEXT }
+            }
+            detections {
+              score: 0.9
+              signal_type: { standard: TEXT }
+            }
+          )"));
+
+  runner->MutableInputs()->Index(2).packets.push_back(
+      Adopt(input_ocr.release()).At(Timestamp(0)));
+
+  MP_ASSERT_OK(runner->Run());
+
+  const std::vector<Packet>& output_packets =
+      runner->Outputs().Index(0).packets;
+  const auto& detection_set = output_packets[0].Get<DetectionSet>();
+
+  ASSERT_EQ(detection_set.detections().size(), 4);
+  EXPECT_FLOAT_EQ(detection_set.detections(0).score(), .55);
+  EXPECT_FLOAT_EQ(detection_set.detections(1).score(), .53);
+  EXPECT_FLOAT_EQ(detection_set.detections(2).score(), .93);
+  EXPECT_FLOAT_EQ(detection_set.detections(3).score(), .99);
+}
+
+TEST(SignalFusingCalculatorTest, ThreeInputTracking) {
+  auto runner = absl::make_unique<CalculatorRunner>(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfigB));
+
+  auto input_border_0 = absl::make_unique<bool>(false);
+  runner->MutableInputs()->Index(0).packets.push_back(
+      Adopt(input_border_0.release()).At(Timestamp(0)));
+
+  // Time zero.
+  auto input_face_0 =
+      absl::make_unique<DetectionSet>(ParseTextProtoOrDie<DetectionSet>(
+          R"(
+            detections {
+              score: 0.2
+              signal_type: { standard: FACE_FULL }
+              tracking_id: 0
+            }
+            detections {
+              score: 0.0
+              signal_type: { standard: FACE_FULL }
+              tracking_id: 1
+            }
+            detections {
+              score: 0.1
+              signal_type: { standard: FACE_FULL }
+            }
+          )"));
+
+  runner->MutableInputs()->Index(1).packets.push_back(
+      Adopt(input_face_0.release()).At(Timestamp(0)));
+
+  auto input_ocr_0 =
+      absl::make_unique<DetectionSet>(ParseTextProtoOrDie<DetectionSet>(
+          R"(
+            detections {
+              score: 0.2
+              signal_type: { custom: "text" }
+            }
+          )"));
+
+  runner->MutableInputs()->Index(2).packets.push_back(
+      Adopt(input_ocr_0.release()).At(Timestamp(0)));
+
+  auto input_agn_0 =
+      absl::make_unique<DetectionSet>(ParseTextProtoOrDie<DetectionSet>(
+          R"(
+            detections {
+              score: 0.3
+              signal_type: { standard: LOGO }
+              tracking_id: 0
+            }
+          )"));
+
+  runner->MutableInputs()->Index(3).packets.push_back(
+      Adopt(input_agn_0.release()).At(Timestamp(0)));
+
+  // Time one
+  auto input_border_1 = absl::make_unique<bool>(false);
+  runner->MutableInputs()->Index(0).packets.push_back(
+      Adopt(input_border_1.release()).At(Timestamp(1)));
+
+  auto input_face_1 =
+      absl::make_unique<DetectionSet>(ParseTextProtoOrDie<DetectionSet>(
+          R"(
+            detections {
+              score: 0.7
+              signal_type: { standard: FACE_FULL }
+              tracking_id: 0
+            }
+            detections {
+              score: 0.9
+              signal_type: { standard: FACE_FULL }
+              tracking_id: 1
+            }
+            detections {
+              score: 0.2
+              signal_type: { standard: FACE_FULL }
+            }
+          )"));
+
+  runner->MutableInputs()->Index(1).packets.push_back(
+      Adopt(input_face_1.release()).At(Timestamp(1)));
+
+  auto input_ocr_1 =
+      absl::make_unique<DetectionSet>(ParseTextProtoOrDie<DetectionSet>(
+          R"(
+            detections {
+              score: 0.3
+              signal_type: { custom: "text" }
+            }
+          )"));
+
+  runner->MutableInputs()->Index(2).packets.push_back(
+      Adopt(input_ocr_1.release()).At(Timestamp(1)));
+
+  auto input_agn_1 =
+      absl::make_unique<DetectionSet>(ParseTextProtoOrDie<DetectionSet>(
+          R"(
+            detections {
+              score: 0.3
+              signal_type: { standard: LOGO }
+              tracking_id: 0
+            }
+          )"));
+
+  runner->MutableInputs()->Index(3).packets.push_back(
+      Adopt(input_agn_1.release()).At(Timestamp(1)));
+
+  // Time two
+  auto input_border_2 = absl::make_unique<bool>(false);
+  runner->MutableInputs()->Index(0).packets.push_back(
+      Adopt(input_border_2.release()).At(Timestamp(2)));
+
+  auto input_face_2 =
+      absl::make_unique<DetectionSet>(ParseTextProtoOrDie<DetectionSet>(
+          R"(
+            detections {
+              score: 0.8
+              signal_type: { standard: FACE_FULL }
+              tracking_id: 0
+            }
+            detections {
+              score: 0.9
+              signal_type: { standard: FACE_FULL }
+              tracking_id: 1
+            }
+            detections {
+              score: 0.3
+              signal_type: { standard: FACE_FULL }
+            }
+          )"));
+
+  runner->MutableInputs()->Index(1).packets.push_back(
+      Adopt(input_face_2.release()).At(Timestamp(2)));
+
+  auto input_ocr_2 =
+      absl::make_unique<DetectionSet>(ParseTextProtoOrDie<DetectionSet>(
+          R"(
+            detections {
+              score: 0.3
+              signal_type: { custom: "text" }
+            }
+          )"));
+
+  runner->MutableInputs()->Index(2).packets.push_back(
+      Adopt(input_ocr_2.release()).At(Timestamp(2)));
+
+  auto input_agn_2 =
+      absl::make_unique<DetectionSet>(ParseTextProtoOrDie<DetectionSet>(
+          R"(
+            detections {
+              score: 0.9
+              signal_type: { standard: LOGO }
+              tracking_id: 0
+            }
+          )"));
+
+  runner->MutableInputs()->Index(3).packets.push_back(
+      Adopt(input_agn_2.release()).At(Timestamp(2)));
+
+  // Time three (new scene)
+  auto input_border_3 = absl::make_unique<bool>(true);
+  runner->MutableInputs()->Index(0).packets.push_back(
+      Adopt(input_border_3.release()).At(Timestamp(3)));
+
+  auto input_face_3 =
+      absl::make_unique<DetectionSet>(ParseTextProtoOrDie<DetectionSet>(
+          R"(
+            detections {
+              score: 0.2
+              signal_type: { standard: FACE_FULL }
+              tracking_id: 0
+            }
+            detections {
+              score: 0.3
+              signal_type: { standard: FACE_FULL }
+              tracking_id: 1
+            }
+            detections {
+              score: 0.4
+              signal_type: { standard: FACE_FULL }
+            }
+          )"));
+
+  runner->MutableInputs()->Index(1).packets.push_back(
+      Adopt(input_face_3.release()).At(Timestamp(3)));
+
+  auto input_ocr_3 =
+      absl::make_unique<DetectionSet>(ParseTextProtoOrDie<DetectionSet>(
+          R"(
+            detections {
+              score: 0.5
+              signal_type: { custom: "text" }
+            }
+          )"));
+
+  runner->MutableInputs()->Index(2).packets.push_back(
+      Adopt(input_ocr_3.release()).At(Timestamp(3)));
+
+  auto input_agn_3 =
+      absl::make_unique<DetectionSet>(ParseTextProtoOrDie<DetectionSet>(
+          R"(
+            detections {
+              score: 0.6
+              signal_type: { standard: LOGO }
+              tracking_id: 0
+            }
+          )"));
+
+  runner->MutableInputs()->Index(3).packets.push_back(
+      Adopt(input_agn_3.release()).At(Timestamp(3)));
+
+  MP_ASSERT_OK(runner->Run());
+
+  // Check time 0
+  std::vector<Packet> output_packets = runner->Outputs().Index(0).packets;
+  DetectionSet detection_set = output_packets[0].Get<DetectionSet>();
+
+  float face_id_0 = (.2 + .7 + .8) / 3;
+  face_id_0 = face_id_0 * .1 + .5;
+  float face_id_1 = (0.0 + .9 + .9) / 3;
+  face_id_1 = face_id_1 * .1 + .5;
+  float face_3 = 0.1;
+  face_3 = face_3 * .1 + .5;
+  float ocr_1 = 0.2;
+  ocr_1 = ocr_1 * .1 + .9;
+  float agn_1 = (.3 + .3 + .9) / 3;
+  agn_1 = agn_1 * .2 + .1;
+
+  ASSERT_EQ(detection_set.detections().size(), 5);
+  EXPECT_FLOAT_EQ(detection_set.detections(0).score(), face_id_0);
+  EXPECT_FLOAT_EQ(detection_set.detections(1).score(), face_id_1);
+  EXPECT_FLOAT_EQ(detection_set.detections(2).score(), face_3);
+  EXPECT_FLOAT_EQ(detection_set.detections(3).score(), ocr_1);
+  EXPECT_FLOAT_EQ(detection_set.detections(4).score(), agn_1);
+
+  // Check time 1
+  detection_set = output_packets[1].Get<DetectionSet>();
+
+  face_id_0 = (.2 + .7 + .8) / 3;
+  face_id_0 = face_id_0 * .1 + .5;
+  face_id_1 = (0.0 + .9 + .9) / 3;
+  face_id_1 = face_id_1 * .1 + .5;
+  face_3 = 0.2;
+  face_3 = face_3 * .1 + .5;
+  ocr_1 = 0.3;
+  ocr_1 = ocr_1 * .1 + .9;
+  agn_1 = (.3 + .3 + .9) / 3;
+  agn_1 = agn_1 * .2 + .1;
+
+  ASSERT_EQ(detection_set.detections().size(), 5);
+  EXPECT_FLOAT_EQ(detection_set.detections(0).score(), face_id_0);
+  EXPECT_FLOAT_EQ(detection_set.detections(1).score(), face_id_1);
+  EXPECT_FLOAT_EQ(detection_set.detections(2).score(), face_3);
+  EXPECT_FLOAT_EQ(detection_set.detections(3).score(), ocr_1);
+  EXPECT_FLOAT_EQ(detection_set.detections(4).score(), agn_1);
+
+  // Check time 2
+  detection_set = output_packets[2].Get<DetectionSet>();
+
+  face_id_0 = (.2 + .7 + .8) / 3;
+  face_id_0 = face_id_0 * .1 + .5;
+  face_id_1 = (0.0 + .9 + .9) / 3;
+  face_id_1 = face_id_1 * .1 + .5;
+  face_3 = 0.3;
+  face_3 = face_3 * .1 + .5;
+  ocr_1 = 0.3;
+  ocr_1 = ocr_1 * .1 + .9;
+  agn_1 = (.3 + .3 + .9) / 3;
+  agn_1 = agn_1 * .2 + .1;
+
+  ASSERT_EQ(detection_set.detections().size(), 5);
+  EXPECT_FLOAT_EQ(detection_set.detections(0).score(), face_id_0);
+  EXPECT_FLOAT_EQ(detection_set.detections(1).score(), face_id_1);
+  EXPECT_FLOAT_EQ(detection_set.detections(2).score(), face_3);
+  EXPECT_FLOAT_EQ(detection_set.detections(3).score(), ocr_1);
+  EXPECT_FLOAT_EQ(detection_set.detections(4).score(), agn_1);
+
+  // Check time 3 (new scene)
+  detection_set = output_packets[3].Get<DetectionSet>();
+
+  face_id_0 = 0.2;
+  face_id_0 = face_id_0 * .1 + .5;
+  face_id_1 = 0.3;
+  face_id_1 = face_id_1 * .1 + .5;
+  face_3 = 0.4;
+  face_3 = face_3 * .1 + .5;
+  ocr_1 = 0.5;
+  ocr_1 = ocr_1 * .1 + .9;
+  agn_1 = .6;
+  agn_1 = agn_1 * .2 + .1;
+
+  ASSERT_EQ(detection_set.detections().size(), 5);
+  EXPECT_FLOAT_EQ(detection_set.detections(0).score(), face_id_0);
+  EXPECT_FLOAT_EQ(detection_set.detections(1).score(), face_id_1);
+  EXPECT_FLOAT_EQ(detection_set.detections(2).score(), face_3);
+  EXPECT_FLOAT_EQ(detection_set.detections(3).score(), ocr_1);
+  EXPECT_FLOAT_EQ(detection_set.detections(4).score(), agn_1);
+}
+
+}  // namespace
+}  // namespace autoflip
+}  // namespace mediapipe

mediapipe/examples/desktop/autoflip/calculators/testdata/BUILD

@@ -0,0 +1,23 @@
+# Copyright 2019 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+licenses(["notice"])  # Apache 2.0
+
+filegroup(
+    name = "test_images",
+    srcs = [
+        "dino.jpg",
+    ],
+    visibility = ["//visibility:public"],
+)

mediapipe/examples/desktop/autoflip/calculators/video_filtering_calculator.cc

@@ -0,0 +1,121 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <string>
+
+#include "absl/strings/string_view.h"
+#include "absl/strings/substitute.h"
+#include "mediapipe/examples/desktop/autoflip/calculators/video_filtering_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/image_frame.h"
+#include "mediapipe/framework/formats/image_frame_opencv.h"
+#include "mediapipe/framework/port/status_builder.h"
+
+namespace mediapipe {
+namespace autoflip {
+namespace {
+constexpr char kInputFrameTag[] = "INPUT_FRAMES";
+constexpr char kOutputFrameTag[] = "OUTPUT_FRAMES";
+}  // namespace
+
+// This calculator filters out frames based on criteria specified in the
+// options. One use case is to filter based on the aspect ratio. Future work
+// can implement more filter types.
+//
+// Input: Video frames.
+// Output: Video frames that pass all filters.
+//
+// Example config:
+// node {
+//   calculator: "VideoFilteringCalculator"
+//   input_stream: "INPUT_FRAMES:frames"
+//   output_stream: "OUTPUT_FRAMES:output_frames"
+//   options: {
+//     [mediapipe.autoflip.VideoFilteringCalculatorOptions.ext]: {
+//       fail_if_any: true
+//       aspect_ratio_filter {
+//         target_width: 400
+//         target_height: 600
+//         filter_type: UPPER_ASPECT_RATIO_THRESHOLD
+//       }
+//     }
+//   }
+// }
+class VideoFilteringCalculator : public CalculatorBase {
+ public:
+  VideoFilteringCalculator() = default;
+  ~VideoFilteringCalculator() override = default;
+
+  static ::mediapipe::Status GetContract(CalculatorContract* cc);
+
+  ::mediapipe::Status Process(CalculatorContext* cc) override;
+};
+REGISTER_CALCULATOR(VideoFilteringCalculator);
+
+::mediapipe::Status VideoFilteringCalculator::GetContract(
+    CalculatorContract* cc) {
+  cc->Inputs().Tag(kInputFrameTag).Set<ImageFrame>();
+  cc->Outputs().Tag(kOutputFrameTag).Set<ImageFrame>();
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status VideoFilteringCalculator::Process(CalculatorContext* cc) {
+  const auto& options = cc->Options<VideoFilteringCalculatorOptions>();
+
+  const Packet& input_packet = cc->Inputs().Tag(kInputFrameTag).Value();
+  const ImageFrame& frame = input_packet.Get<ImageFrame>();
+
+  RET_CHECK(options.has_aspect_ratio_filter());
+  const auto filter_type = options.aspect_ratio_filter().filter_type();
+  RET_CHECK_NE(
+      filter_type,
+      VideoFilteringCalculatorOptions::AspectRatioFilter::UNKNOWN_FILTER_TYPE);
+  if (filter_type ==
+      VideoFilteringCalculatorOptions::AspectRatioFilter::NO_FILTERING) {
+    cc->Outputs().Tag(kOutputFrameTag).AddPacket(input_packet);
+    return ::mediapipe::OkStatus();
+  }
+  const int target_width = options.aspect_ratio_filter().target_width();
+  const int target_height = options.aspect_ratio_filter().target_height();
+  RET_CHECK_GT(target_width, 0);
+  RET_CHECK_GT(target_height, 0);
+
+  bool should_pass = false;
+  cv::Mat frame_mat = ::mediapipe::formats::MatView(&frame);
+  const double ratio = static_cast<double>(frame_mat.cols) / frame_mat.rows;
+  const double target_ratio = static_cast<double>(target_width) / target_height;
+  if (filter_type == VideoFilteringCalculatorOptions::AspectRatioFilter::
+                         UPPER_ASPECT_RATIO_THRESHOLD &&
+      ratio <= target_ratio) {
+    should_pass = true;
+  } else if (filter_type == VideoFilteringCalculatorOptions::AspectRatioFilter::
+                                LOWER_ASPECT_RATIO_THRESHOLD &&
+             ratio >= target_ratio) {
+    should_pass = true;
+  }
+  if (should_pass) {
+    cc->Outputs().Tag(kOutputFrameTag).AddPacket(input_packet);
+    return ::mediapipe::OkStatus();
+  }
+  if (options.fail_if_any()) {
+    return ::mediapipe::UnknownErrorBuilder(MEDIAPIPE_LOC) << absl::Substitute(
+               "Failing due to aspect ratio. Target aspect ratio: $0. Frame "
+               "width: $1, height: $2.",
+               target_ratio, frame.Width(), frame.Height());
+  }
+
+  return ::mediapipe::OkStatus();
+}
+}  // namespace autoflip
+}  // namespace mediapipe

mediapipe/examples/desktop/autoflip/calculators/video_filtering_calculator.proto

@@ -0,0 +1,56 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe.autoflip;
+
+import "mediapipe/framework/calculator.proto";
+
+message VideoFilteringCalculatorOptions {
+  extend mediapipe.CalculatorOptions {
+    optional VideoFilteringCalculatorOptions ext = 278504113;
+  }
+
+  // If true, when an input frame needs be filtered out according to the filter
+  // type and conditions, the calculator would return a FAIL status. Otherwise,
+  // the calculator would simply skip the filtered frames and would not pass it
+  // down to downstream nodes.
+  optional bool fail_if_any = 1 [default = false];
+
+  message AspectRatioFilter {
+    // Target width and height, which define the aspect ratio
+    // (i.e. target_width / target_height) to compare input frames with. The
+    // actual values of these fields do not matter, only the ratio between them
+    // does. These values must be set to positive.
+    optional int32 target_width = 1 [default = -1];
+    optional int32 target_height = 2 [default = -1];
+    enum FilterType {
+      UNKNOWN_FILTER_TYPE = 0;
+      // Use this type when the target width and height defines an upper bound
+      // (inclusive) of the aspect ratio.
+      UPPER_ASPECT_RATIO_THRESHOLD = 1;
+      // Use this type when the target width and height defines a lower bound
+      // (inclusive) of the aspect ratio.
+      LOWER_ASPECT_RATIO_THRESHOLD = 2;
+      // Use this type to configure the calculator as a no-op pass-through node.
+      NO_FILTERING = 3;
+    }
+    optional FilterType filter_type = 3;
+  }
+
+  oneof filter {
+    AspectRatioFilter aspect_ratio_filter = 2;
+  }
+}

mediapipe/examples/desktop/autoflip/calculators/video_filtering_calculator_test.cc

@@ -0,0 +1,177 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "absl/strings/string_view.h"
+#include "absl/strings/substitute.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/formats/image_frame.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/status_builder.h"
+#include "mediapipe/framework/port/status_matchers.h"
+
+namespace mediapipe {
+namespace autoflip {
+namespace {
+
+// Default configuration of the calculator.
+CalculatorGraphConfig::Node GetCalculatorNode(
+    const std::string& fail_if_any, const std::string& extra_options = "") {
+  return ParseTextProtoOrDie<CalculatorGraphConfig::Node>(
+      absl::Substitute(R"(
+        calculator: "VideoFilteringCalculator"
+        input_stream: "INPUT_FRAMES:frames"
+        output_stream: "OUTPUT_FRAMES:output_frames"
+        options: {
+          [mediapipe.autoflip.VideoFilteringCalculatorOptions.ext]: {
+            fail_if_any: $0
+            $1
+          }
+        }
+    )",
+                       fail_if_any, extra_options));
+}
+
+TEST(VideoFilterCalculatorTest, UpperBoundNoPass) {
+  CalculatorGraphConfig::Node config = GetCalculatorNode("false", R"(
+    aspect_ratio_filter {
+      target_width: 2
+      target_height: 1
+      filter_type: UPPER_ASPECT_RATIO_THRESHOLD
+    }
+  )");
+
+  auto runner = ::absl::make_unique<CalculatorRunner>(config);
+  const int kFixedWidth = 1000;
+  const double kAspectRatio = 5.0 / 1.0;
+  auto input_frame = ::absl::make_unique<ImageFrame>(
+      ImageFormat::SRGB, kFixedWidth,
+      static_cast<int>(kFixedWidth / kAspectRatio), 16);
+  runner->MutableInputs()
+      ->Tag("INPUT_FRAMES")
+      .packets.push_back(Adopt(input_frame.release()).At(Timestamp(1000)));
+  MP_ASSERT_OK(runner->Run());
+  const auto& output_packet = runner->Outputs().Tag("OUTPUT_FRAMES").packets;
+  EXPECT_TRUE(output_packet.empty());
+}
+
+TEST(VerticalFrameRemovalCalculatorTest, UpperBoundPass) {
+  CalculatorGraphConfig::Node config = GetCalculatorNode("false", R"(
+    aspect_ratio_filter {
+      target_width: 2
+      target_height: 1
+      filter_type: UPPER_ASPECT_RATIO_THRESHOLD
+    }
+  )");
+
+  auto runner = ::absl::make_unique<CalculatorRunner>(config);
+  const int kWidth = 1000;
+  const double kAspectRatio = 1.0 / 5.0;
+  const double kHeight = static_cast<int>(kWidth / kAspectRatio);
+  auto input_frame =
+      ::absl::make_unique<ImageFrame>(ImageFormat::SRGB, kWidth, kHeight, 16);
+  runner->MutableInputs()
+      ->Tag("INPUT_FRAMES")
+      .packets.push_back(Adopt(input_frame.release()).At(Timestamp(1000)));
+  MP_ASSERT_OK(runner->Run());
+  const auto& output_packet = runner->Outputs().Tag("OUTPUT_FRAMES").packets;
+  EXPECT_EQ(1, output_packet.size());
+  auto& output_frame = output_packet[0].Get<ImageFrame>();
+  EXPECT_EQ(kWidth, output_frame.Width());
+  EXPECT_EQ(kHeight, output_frame.Height());
+}
+
+TEST(VideoFilterCalculatorTest, LowerBoundNoPass) {
+  CalculatorGraphConfig::Node config = GetCalculatorNode("false", R"(
+    aspect_ratio_filter {
+      target_width: 2
+      target_height: 1
+      filter_type: LOWER_ASPECT_RATIO_THRESHOLD
+    }
+  )");
+
+  auto runner = ::absl::make_unique<CalculatorRunner>(config);
+  const int kFixedWidth = 1000;
+  const double kAspectRatio = 1.0 / 1.0;
+  auto input_frame = ::absl::make_unique<ImageFrame>(
+      ImageFormat::SRGB, kFixedWidth,
+      static_cast<int>(kFixedWidth / kAspectRatio), 16);
+  runner->MutableInputs()
+      ->Tag("INPUT_FRAMES")
+      .packets.push_back(Adopt(input_frame.release()).At(Timestamp(1000)));
+  MP_ASSERT_OK(runner->Run());
+  const auto& output_packet = runner->Outputs().Tag("OUTPUT_FRAMES").packets;
+  EXPECT_TRUE(output_packet.empty());
+}
+
+TEST(VerticalFrameRemovalCalculatorTest, LowerBoundPass) {
+  CalculatorGraphConfig::Node config = GetCalculatorNode("false", R"(
+    aspect_ratio_filter {
+      target_width: 2
+      target_height: 1
+      filter_type: LOWER_ASPECT_RATIO_THRESHOLD
+    }
+  )");
+
+  auto runner = ::absl::make_unique<CalculatorRunner>(config);
+  const int kWidth = 1000;
+  const double kAspectRatio = 5.0 / 1.0;
+  const double kHeight = static_cast<int>(kWidth / kAspectRatio);
+  auto input_frame =
+      ::absl::make_unique<ImageFrame>(ImageFormat::SRGB, kWidth, kHeight, 16);
+  runner->MutableInputs()
+      ->Tag("INPUT_FRAMES")
+      .packets.push_back(Adopt(input_frame.release()).At(Timestamp(1000)));
+  MP_ASSERT_OK(runner->Run());
+  const auto& output_packet = runner->Outputs().Tag("OUTPUT_FRAMES").packets;
+  EXPECT_EQ(1, output_packet.size());
+  auto& output_frame = output_packet[0].Get<ImageFrame>();
+  EXPECT_EQ(kWidth, output_frame.Width());
+  EXPECT_EQ(kHeight, output_frame.Height());
+}
+
+// Test that an error should be generated when fail_if_any is true.
+TEST(VerticalFrameRemovalCalculatorTest, OutputError) {
+  CalculatorGraphConfig::Node config = GetCalculatorNode("true", R"(
+    aspect_ratio_filter {
+      target_width: 2
+      target_height: 1
+      filter_type: LOWER_ASPECT_RATIO_THRESHOLD
+    }
+  )");
+
+  auto runner = ::absl::make_unique<CalculatorRunner>(config);
+  const int kFixedWidth = 1000;
+  const double kAspectRatio = 1.0 / 1.0;
+  auto input_frame = ::absl::make_unique<ImageFrame>(
+      ImageFormat::SRGB, kFixedWidth,
+      static_cast<int>(kFixedWidth / kAspectRatio), 16);
+  runner->MutableInputs()
+      ->Tag("INPUT_FRAMES")
+      .packets.push_back(Adopt(input_frame.release()).At(Timestamp(1000)));
+  ::mediapipe::Status status = runner->Run();
+  EXPECT_EQ(status.code(), ::mediapipe::StatusCode::kUnknown);
+  EXPECT_THAT(status.ToString(),
+              ::testing::HasSubstr("Failing due to aspect ratio"));
+}
+
+}  // namespace
+}  // namespace autoflip
+}  // namespace mediapipe

mediapipe/examples/desktop/autoflip/quality/BUILD

@@ -0,0 +1,331 @@
+load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
+
+# Copyright 2019 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+licenses(["notice"])  # Apache 2.0
+
+package(default_visibility = ["//mediapipe/examples:__subpackages__"])
+
+proto_library(
+    name = "cropping_proto",
+    srcs = ["cropping.proto"],
+    deps = [
+        "//mediapipe/examples/desktop/autoflip:autoflip_messages_proto",
+    ],
+)
+
+mediapipe_cc_proto_library(
+    name = "cropping_cc_proto",
+    srcs = ["cropping.proto"],
+    cc_deps = ["//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto"],
+    visibility = ["//mediapipe/examples:__subpackages__"],
+    deps = [":cropping_proto"],
+)
+
+proto_library(
+    name = "focus_point_proto",
+    srcs = ["focus_point.proto"],
+)
+
+mediapipe_cc_proto_library(
+    name = "focus_point_cc_proto",
+    srcs = ["focus_point.proto"],
+    visibility = ["//mediapipe/examples:__subpackages__"],
+    deps = [":focus_point_proto"],
+)
+
+cc_library(
+    name = "frame_crop_region_computer",
+    srcs = ["frame_crop_region_computer.cc"],
+    hdrs = ["frame_crop_region_computer.h"],
+    deps = [
+        ":cropping_cc_proto",
+        ":utils",
+        "//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+    ],
+)
+
+cc_library(
+    name = "math_utils",
+    hdrs = ["math_utils.h"],
+)
+
+cc_library(
+    name = "piecewise_linear_function",
+    srcs = ["piecewise_linear_function.cc"],
+    hdrs = ["piecewise_linear_function.h"],
+    deps = [
+        "//mediapipe/framework/port:status",
+    ],
+)
+
+cc_library(
+    name = "padding_effect_generator",
+    srcs = ["padding_effect_generator.cc"],
+    hdrs = ["padding_effect_generator.h"],
+    deps = [
+        "//mediapipe/framework/formats:image_frame",
+        "//mediapipe/framework/formats:image_frame_opencv",
+        "//mediapipe/framework/port:commandlineflags",
+        "//mediapipe/framework/port:opencv_core",
+        "//mediapipe/framework/port:opencv_imgproc",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+    ],
+)
+
+cc_library(
+    name = "scene_camera_motion_analyzer",
+    srcs = ["scene_camera_motion_analyzer.cc"],
+    hdrs = ["scene_camera_motion_analyzer.h"],
+    deps = [
+        ":cropping_cc_proto",
+        ":focus_point_cc_proto",
+        ":math_utils",
+        ":piecewise_linear_function",
+        ":utils",
+        "//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto",
+        "//mediapipe/framework:timestamp",
+        "//mediapipe/framework/port:integral_types",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/memory",
+        "@com_google_absl//absl/strings",
+        "@com_google_absl//absl/strings:str_format",
+    ],
+)
+
+cc_library(
+    name = "scene_cropping_viz",
+    srcs = ["scene_cropping_viz.cc"],
+    hdrs = ["scene_cropping_viz.h"],
+    deps = [
+        ":cropping_cc_proto",
+        ":focus_point_cc_proto",
+        "//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:image_format_cc_proto",
+        "//mediapipe/framework/formats:image_frame",
+        "//mediapipe/framework/formats:image_frame_opencv",
+        "//mediapipe/framework/port:opencv_core",
+        "//mediapipe/framework/port:opencv_imgproc",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/memory",
+    ],
+)
+
+cc_library(
+    name = "polynomial_regression_path_solver",
+    srcs = ["polynomial_regression_path_solver.cc"],
+    hdrs = ["polynomial_regression_path_solver.h"],
+    deps = [
+        ":focus_point_cc_proto",
+        "//mediapipe/framework/port:opencv_core",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "@ceres_solver//:ceres",
+    ],
+)
+
+cc_library(
+    name = "scene_cropper",
+    srcs = ["scene_cropper.cc"],
+    hdrs = ["scene_cropper.h"],
+    deps = [
+        ":cropping_cc_proto",
+        ":focus_point_cc_proto",
+        ":polynomial_regression_path_solver",
+        ":utils",
+        "//mediapipe/framework/port:opencv_core",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/memory",
+    ],
+)
+
+cc_library(
+    name = "utils",
+    srcs = ["utils.cc"],
+    hdrs = ["utils.h"],
+    deps = [
+        ":cropping_cc_proto",
+        ":math_utils",
+        ":piecewise_linear_function",
+        "//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto",
+        "//mediapipe/framework/port:opencv_core",
+        "//mediapipe/framework/port:opencv_imgproc",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/memory",
+    ],
+)
+
+cc_test(
+    name = "frame_crop_region_computer_test",
+    srcs = ["frame_crop_region_computer_test.cc"],
+    deps = [
+        ":cropping_cc_proto",
+        ":frame_crop_region_computer",
+        "//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto",
+        "//mediapipe/framework/port:gtest_main",
+        "@com_google_absl//absl/memory",
+    ],
+)
+
+cc_test(
+    name = "piecewise_linear_function_test",
+    srcs = ["piecewise_linear_function_test.cc"],
+    deps = [
+        ":piecewise_linear_function",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:status",
+    ],
+)
+
+cc_test(
+    name = "scene_camera_motion_analyzer_test",
+    srcs = ["scene_camera_motion_analyzer_test.cc"],
+    data = [
+        "//mediapipe/examples/desktop/autoflip/quality/testdata:camera_motion_tracking_scene_frame_results.csv",
+    ],
+    deps = [
+        ":focus_point_cc_proto",
+        ":piecewise_linear_function",
+        ":scene_camera_motion_analyzer",
+        "//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto",
+        "//mediapipe/framework/deps:file_path",
+        "//mediapipe/framework/port:file_helpers",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
+cc_test(
+    name = "padding_effect_generator_test",
+    srcs = ["padding_effect_generator_test.cc"],
+    data = [
+        "//mediapipe/examples/desktop/autoflip/quality/testdata:google.jpg",
+        "//mediapipe/examples/desktop/autoflip/quality/testdata:result_0.3.jpg",
+        "//mediapipe/examples/desktop/autoflip/quality/testdata:result_0.3_solid_background.jpg",
+        "//mediapipe/examples/desktop/autoflip/quality/testdata:result_0.6.jpg",
+        "//mediapipe/examples/desktop/autoflip/quality/testdata:result_0.6_solid_background.jpg",
+        "//mediapipe/examples/desktop/autoflip/quality/testdata:result_1.6.jpg",
+        "//mediapipe/examples/desktop/autoflip/quality/testdata:result_1.6_solid_background.jpg",
+        "//mediapipe/examples/desktop/autoflip/quality/testdata:result_1.jpg",
+        "//mediapipe/examples/desktop/autoflip/quality/testdata:result_1_solid_background.jpg",
+        "//mediapipe/examples/desktop/autoflip/quality/testdata:result_2.5.jpg",
+        "//mediapipe/examples/desktop/autoflip/quality/testdata:result_2.5_solid_background.jpg",
+        "//mediapipe/examples/desktop/autoflip/quality/testdata:result_3.4.jpg",
+        "//mediapipe/examples/desktop/autoflip/quality/testdata:result_3.4_solid_background.jpg",
+    ],
+    deps = [
+        ":padding_effect_generator",
+        "//mediapipe/framework/deps:file_path",
+        "//mediapipe/framework/formats:image_frame",
+        "//mediapipe/framework/formats:image_frame_opencv",
+        "//mediapipe/framework/port:commandlineflags",
+        "//mediapipe/framework/port:file_helpers",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:opencv_imgcodecs",
+        "//mediapipe/framework/port:opencv_imgproc",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
+cc_test(
+    name = "polynomial_regression_path_solver_test",
+    srcs = ["polynomial_regression_path_solver_test.cc"],
+    deps = [
+        ":focus_point_cc_proto",
+        ":polynomial_regression_path_solver",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:opencv_core",
+        "//mediapipe/framework/port:status",
+    ],
+)
+
+cc_test(
+    name = "scene_cropper_test",
+    size = "small",
+    timeout = "short",
+    srcs = ["scene_cropper_test.cc"],
+    deps = [
+        ":focus_point_cc_proto",
+        ":scene_cropper",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:opencv_core",
+    ],
+)
+
+cc_test(
+    name = "utils_test",
+    srcs = ["utils_test.cc"],
+    deps = [
+        ":cropping_cc_proto",
+        ":utils",
+        "//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:integral_types",
+        "//mediapipe/framework/port:opencv_core",
+    ],
+)
+
+proto_library(
+    name = "visual_scorer_proto",
+    srcs = ["visual_scorer.proto"],
+)
+
+mediapipe_cc_proto_library(
+    name = "visual_scorer_cc_proto",
+    srcs = ["visual_scorer.proto"],
+    visibility = ["//mediapipe/examples:__subpackages__"],
+    deps = [":visual_scorer_proto"],
+)
+
+cc_library(
+    name = "visual_scorer",
+    srcs = ["visual_scorer.cc"],
+    hdrs = ["visual_scorer.h"],
+    deps = [
+        ":visual_scorer_cc_proto",
+        "//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto",
+        "//mediapipe/framework/port:opencv_core",
+        "//mediapipe/framework/port:opencv_imgproc",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+    ],
+)
+
+cc_test(
+    name = "visual_scorer_test",
+    srcs = [
+        "visual_scorer_test.cc",
+    ],
+    linkstatic = 1,
+    deps = [
+        ":visual_scorer",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:opencv_core",
+        "//mediapipe/framework/port:opencv_imgproc",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/port:status",
+    ],
+)

mediapipe/examples/desktop/autoflip/quality/cropping.proto

@@ -0,0 +1,217 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe.autoflip;
+
+import "mediapipe/examples/desktop/autoflip/autoflip_messages.proto";
+
+// All relevant information for key frames, including timestamp and detected
+// features. This object should be generated by calling PackKeyFrameInfo() in
+// the util namespace. It is passed in to ComputeFrameCropRegion().
+message KeyFrameInfo {
+  // Frame timestamp (in microseconds).
+  optional int64 timestamp_ms = 1;
+  // Detected features.
+  optional DetectionSet detections = 2;
+}
+
+// User-specified key frame crop options (such as target width and height).
+message KeyFrameCropOptions {
+  // Target crop size.
+  // Note: if you are using the SceneCroppingCalculator, DO NOT set these fields
+  // manually as they will be then overwritten inside the calculator.
+  optional int32 target_width = 1;
+  optional int32 target_height = 2;
+  // Option for how region score is aggregated from individual feature scores.
+  // TODO: consider merging this enum type into the signal fusing
+  // calculator.
+  enum ScoreAggregationType {
+    // Unknown value (should not be used).
+    UNKNOWN = 0;
+    // Takes the score of the feature with maximum score.
+    MAXIMUM = 1;
+    // Takes the sum of the scores of the required regions.
+    SUM_REQUIRED = 2;
+    // Takes the sum of the scores of all the regions that are fully covered.
+    SUM_ALL = 3;
+    // Uses a constant score 1.0 for all crop regions.
+    CONSTANT = 4;
+  }
+  optional ScoreAggregationType score_aggregation_type = 3 [default = SUM_ALL];
+  // Minimum centered coverage fraction (in length, not area) for a non-required
+  // region to be included in the crop region. Applies to both dimensions.
+  optional float non_required_region_min_coverage_fraction = 4 [default = 0.5];
+}
+
+// Key frame crop result containing the crop region rectangle, along with
+// summary information on the cropping, such as whether all required regions
+// could fit inside the target size, and what fraction of non-required regions
+// are fully covered. This object is returned by ComputeFrameCropRegion() in
+// the FrameCropRegionComputer class.
+message KeyFrameCropResult {
+  // Successfully covers all required features. If there are no required
+  // regions, this field is set to true.
+  optional bool are_required_regions_covered_in_target_size = 1;
+  // Fraction of non-required features covered.
+  optional float fraction_non_required_covered = 2;
+  // Whether required crop region is empty (no detections).
+  optional bool required_region_is_empty = 3;
+  // Whether (full) crop region is empty (no detections).
+  optional bool region_is_empty = 4;
+  // Computed required crop region.
+  optional Rect required_region = 5;
+  // Computed (full) crop region.
+  optional Rect region = 6;
+  // Score of the computed crop region based on the detected features.
+  optional float region_score = 7;
+}
+
+// Compact processed scene key frame info containing timestamp, center position,
+// and score. Each key frame has one SceneKeyFrameCompactInfo in
+// SceneKeyFrameCropSummary.
+message SceneKeyFrameCompactInfo {
+  // Key frame timestamp (in microseconds).
+  optional int64 timestamp_ms = 1;
+  // Key frame crop region center in the horizontal/vertical directions (in
+  // pixels).
+  optional float center_x = 2;
+  optional float center_y = 3;
+  // Key frame crop region score.
+  optional float score = 4;
+}
+
+// Summary information for the key frame crop results in a scene. Computed by
+// AnalyzeSceneKeyFrameCropResults() in the SceneCameraMotionAnalyzer class.
+// Used to decide camera motion type and populate salient point frames.
+message SceneKeyFrameCropSummary {
+  // Scene frame size.
+  optional int32 scene_frame_width = 1;
+  optional int32 scene_frame_height = 2;
+
+  // Number of key frames in the scene.
+  optional int32 num_key_frames = 3;
+  // Scene key frame compact infos.
+  repeated SceneKeyFrameCompactInfo key_frame_compact_infos = 4;
+
+  // The minimum/maximum values of key frames' crop centers in the horizontal/
+  // vertical directions.
+  optional float key_frame_center_min_x = 5;
+  optional float key_frame_center_max_x = 6;
+  optional float key_frame_center_min_y = 7;
+  optional float key_frame_center_max_y = 8;
+
+  // The union of all the key frame required crop regions. When camera is steady
+  // the crop window is set to cover this union.
+  optional Rect key_frame_required_crop_region_union = 9;
+
+  // The minimum/maximum scores of key frames' crop regions.
+  optional float key_frame_min_score = 10;
+  optional float key_frame_max_score = 11;
+
+  // Size of the scene's crop window, calculated as the maximum of the target
+  // size and the largest size of the key frames' crop regions in the scene.
+  optional int32 crop_window_width = 12;
+  optional int32 crop_window_height = 13;
+
+  // Indicator for whether the scene has any frame with any salient region.
+  optional bool has_salient_region = 14;
+  // Indicator for whether the scene has any frame with any required salient
+  // region.
+  optional bool has_required_salient_region = 15;
+  // Percentage of key frames that are successfully cropped (i.e. covers all
+  // required regions inside the target size).
+  optional float frame_success_rate = 16;
+  // Amount of motion in the horizontal/vertical direction (i.e. the horizontal/
+  // vertical range of the key frame crop centers' position as a fraction of
+  // frame width/height).
+  optional float horizontal_motion_amount = 17;
+  optional float vertical_motion_amount = 18;
+}
+
+// Scene camera motion determined by the SceneCameraMotionAnalyzer class.
+message SceneCameraMotion {
+  // Camera focuses on a fixed center throughout the scene.
+  message SteadyMotion {
+    // Steady look-at center in horizontal/vertical directions (in pixels).
+    optional float steady_look_at_center_x = 1;
+    optional float steady_look_at_center_y = 2;
+  }
+  // Camera tracks key frame salient region centers.
+  message TrackingMotion {
+    // Fields to be added if necessary.
+  }
+  // Camera sweeps from one point to another.
+  message SweepingMotion {
+    // Starting and ending center positions for camera sweeping in pixels.
+    optional float sweep_start_center_x = 1;
+    optional float sweep_start_center_y = 2;
+    optional float sweep_end_center_x = 3;
+    optional float sweep_end_center_y = 4;
+  }
+  oneof motion_type {
+    SteadyMotion steady_motion = 1;
+    TrackingMotion tracking_motion = 2;
+    SweepingMotion sweeping_motion = 3;
+    // Other types that we might support later.
+  }
+}
+
+// User-specified options for analyzing scene camera motion from a collection of
+// key frame crop regions.
+message SceneCameraMotionAnalyzerOptions {
+  // If there is small motion within the scene keep the camera steady at the
+  // center.
+  optional float motion_stabilization_threshold_percent = 1 [default = .30];
+  // Snap to center if there is small motion and already focused closed to the
+  // center.
+  optional float snap_center_max_distance_percent = 2 [default = .08];
+  // Maximum weight for a constraint. Scales scores accordingly so that the
+  // maximum score is equal to this weight.
+  optional float maximum_salient_point_weight = 3 [default = 100.0];
+  // Normalized bound for SalientPoint's in the frame from the border. This is
+  // uniformly applied to the left, right, top, and bottom. It should be
+  // strictly less than 0.5. A narrower bound (closer to 0.5) gives better
+  // constraint enforcement.
+  optional float salient_point_bound = 4 [default = 0.48];
+  // Indicator for whether sweeping is allowed. Note that if a scene can be
+  // seamlessly padded with solid background color, sweeping will be disabled
+  // regardlessly of the value of this flag.
+  optional bool allow_sweeping = 5 [default = true];
+  // Minimal scene time span in seconds to allow camera sweeping.
+  optional float minimum_scene_span_sec_for_sweeping = 6 [default = 1.0];
+  // If success rate in a scene is less than this, then use camera sweeping.
+  optional float minimum_success_rate_for_sweeping = 7 [default = 0.4];
+  // If true, sweep entire frame. Otherwise, sweep the crop window.
+  optional bool sweep_entire_frame = 8 [default = true];
+}
+
+// Video cropping summary information for debugging/statistics.
+message VideoCroppingSummary {
+  message SceneCroppingSummary {
+    // Scene span in seconds.
+    optional float start_sec = 1;
+    optional float end_sec = 2;
+    // Indicator for whether this scene was cut at a real physical scene
+    // boundary (as opposed to force flush).
+    optional bool is_end_of_scene = 3;
+    // Scene camera motion.
+    optional SceneCameraMotion camera_motion = 4;
+    // Indicator for whether the scene is padded.
+    optional bool is_padded = 5;
+  }
+  // Cropping summaries for all the scenes in the video.
+  repeated SceneCroppingSummary scene_summaries = 1;
+}

mediapipe/examples/desktop/autoflip/quality/focus_point.proto

@@ -0,0 +1,79 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe.autoflip;
+
+// Focus point location (normalized w.r.t. frame_width and frame_height, i.e.
+// specified in the domain [0, 1] x [0, 1]).
+
+// For TYPE_INCLUDE:
+// During retargeting and stabilization focus points introduce constraints
+// that will try to keep the normalized location in the rectangle
+// frame_size - normalized bounds.
+// For this soft constraints are used, therefore the weight specifies
+// how "important" the focus point is (higher is better).
+// In particular for each point p the retargeter introduces two pairs of
+// constraints of the form:
+//           x - slack < width - right
+//    and    x + slack > 0 + left,        with slack > 0
+//  where the weight specifies the importance of the slack.
+//
+// For TYPE_EXCLUDE_*:
+// Similar to above, but constraints are introduced to keep
+// the point to the left of the left bound OR the right of the right bound.
+// In particular:
+//          x - slack < left OR
+//          x + slack >= right
+//  Similar to above, the weight specifies the importance of the slack.
+//
+// Note: Choosing a too high weight can lead to
+// jerkiness as the stabilization essentially starts tracking the focus point.
+message FocusPoint {
+  // Normalized location of the point (within domain [0, 1] x [0, 1].
+  optional float norm_point_x = 1 [default = 0.0];
+  optional float norm_point_y = 2 [default = 0.0];
+
+  enum FocusPointType {
+    TYPE_INCLUDE = 1;
+    TYPE_EXCLUDE_LEFT = 2;
+    TYPE_EXCLUDE_RIGHT = 3;
+  }
+
+  // Focus point type. By default we try to frame the focus point within
+  // the bounding box specified by left, bottom, right, top. Alternatively, one
+  // can choose to exclude the point. For details, see discussion above.
+  optional FocusPointType type = 11 [default = TYPE_INCLUDE];
+
+  // Bounds are specified in normalized coordinates [0, 1], FROM the specified
+  // border. Opposing bounds (e.g. left and right) may not add to values
+  // larger than 1.
+  // Default bounds center focus point within centering third of the frame.
+  optional float left = 3 [default = 0.3];
+  optional float bottom = 4 [default = 0.3];
+  optional float right = 9 [default = 0.3];
+  optional float top = 10 [default = 0.3];
+
+  optional float weight = 5 [default = 15];
+
+  extensions 20000 to max;
+}
+
+// Aggregates FocusPoint's for a frame.
+message FocusPointFrame {
+  repeated FocusPoint point = 1;
+
+  extensions 20000 to max;
+}

mediapipe/examples/desktop/autoflip/quality/frame_crop_region_computer.cc

@@ -0,0 +1,259 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/examples/desktop/autoflip/quality/frame_crop_region_computer.h"
+
+#include <cmath>
+
+#include "mediapipe/examples/desktop/autoflip/quality/utils.h"
+#include "mediapipe/framework/port/ret_check.h"
+
+namespace mediapipe {
+namespace autoflip {
+
+::mediapipe::Status FrameCropRegionComputer::ExpandSegmentUnderConstraint(
+    const Segment& segment_to_add, const Segment& base_segment,
+    const int max_length, Segment* combined_segment,
+    CoverType* cover_type) const {
+  RET_CHECK(combined_segment != nullptr) << "Combined segment is null.";
+  RET_CHECK(cover_type != nullptr) << "Cover type is null.";
+
+  const LeftPoint segment_to_add_left = segment_to_add.first;
+  const RightPoint segment_to_add_right = segment_to_add.second;
+  RET_CHECK(segment_to_add_right >= segment_to_add_left)
+      << "Invalid segment to add.";
+  const LeftPoint base_segment_left = base_segment.first;
+  const RightPoint base_segment_right = base_segment.second;
+  RET_CHECK(base_segment_right >= base_segment_left) << "Invalid base segment.";
+  const int base_length = base_segment_right - base_segment_left;
+  RET_CHECK(base_length <= max_length)
+      << "Base segment length exceeds max length.";
+
+  const int segment_to_add_length = segment_to_add_right - segment_to_add_left;
+  const int max_leftout_amount =
+      std::ceil((1.0 - options_.non_required_region_min_coverage_fraction()) *
+                segment_to_add_length / 2);
+  const LeftPoint min_coverage_segment_to_add_left =
+      segment_to_add_left + max_leftout_amount;
+  const LeftPoint min_coverage_segment_to_add_right =
+      segment_to_add_right - max_leftout_amount;
+
+  LeftPoint combined_segment_left =
+      std::min(segment_to_add_left, base_segment_left);
+  RightPoint combined_segment_right =
+      std::max(segment_to_add_right, base_segment_right);
+
+  LeftPoint min_coverage_combined_segment_left =
+      std::min(min_coverage_segment_to_add_left, base_segment_left);
+  RightPoint min_coverage_combined_segment_right =
+      std::max(min_coverage_segment_to_add_right, base_segment_right);
+
+  if ((combined_segment_right - combined_segment_left) <= max_length) {
+    *cover_type = FULLY_COVERED;
+  } else if (min_coverage_combined_segment_right -
+                 min_coverage_combined_segment_left <=
+             max_length) {
+    *cover_type = PARTIALLY_COVERED;
+    combined_segment_left = min_coverage_combined_segment_left;
+    combined_segment_right = min_coverage_combined_segment_right;
+  } else {
+    *cover_type = NOT_COVERED;
+    combined_segment_left = base_segment_left;
+    combined_segment_right = base_segment_right;
+  }
+
+  *combined_segment =
+      std::make_pair(combined_segment_left, combined_segment_right);
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status FrameCropRegionComputer::ExpandRectUnderConstraints(
+    const Rect& rect_to_add, const int max_width, const int max_height,
+    Rect* base_rect, CoverType* cover_type) const {
+  RET_CHECK(base_rect != nullptr) << "Base rect is null.";
+  RET_CHECK(cover_type != nullptr) << "Cover type is null.";
+  RET_CHECK(base_rect->width() <= max_width &&
+            base_rect->height() <= max_height)
+      << "Base rect already exceeds target size.";
+
+  const LeftPoint rect_to_add_left = rect_to_add.x();
+  const RightPoint rect_to_add_right = rect_to_add.x() + rect_to_add.width();
+  const LeftPoint rect_to_add_top = rect_to_add.y();
+  const RightPoint rect_to_add_bottom = rect_to_add.y() + rect_to_add.height();
+  const LeftPoint base_rect_left = base_rect->x();
+  const RightPoint base_rect_right = base_rect->x() + base_rect->width();
+  const LeftPoint base_rect_top = base_rect->y();
+  const RightPoint base_rect_bottom = base_rect->y() + base_rect->height();
+
+  Segment horizontal_combined_segment, vertical_combined_segment;
+  CoverType horizontal_cover_type, vertical_cover_type;
+  const auto horizontal_status = ExpandSegmentUnderConstraint(
+      std::make_pair(rect_to_add_left, rect_to_add_right),
+      std::make_pair(base_rect_left, base_rect_right), max_width,
+      &horizontal_combined_segment, &horizontal_cover_type);
+  MP_RETURN_IF_ERROR(horizontal_status);
+  const auto vertical_status = ExpandSegmentUnderConstraint(
+      std::make_pair(rect_to_add_top, rect_to_add_bottom),
+      std::make_pair(base_rect_top, base_rect_bottom), max_height,
+      &vertical_combined_segment, &vertical_cover_type);
+  MP_RETURN_IF_ERROR(vertical_status);
+
+  if (horizontal_cover_type == NOT_COVERED ||
+      vertical_cover_type == NOT_COVERED) {
+    // Gives up if the segment is not covered in either direction.
+    *cover_type = NOT_COVERED;
+  } else {
+    // Tries to (partially) cover the new rect to be added.
+    base_rect->set_x(horizontal_combined_segment.first);
+    base_rect->set_y(vertical_combined_segment.first);
+    base_rect->set_width(horizontal_combined_segment.second -
+                         horizontal_combined_segment.first);
+    base_rect->set_height(vertical_combined_segment.second -
+                          vertical_combined_segment.first);
+    if (horizontal_cover_type == FULLY_COVERED &&
+        vertical_cover_type == FULLY_COVERED) {
+      *cover_type = FULLY_COVERED;
+    } else {
+      *cover_type = PARTIALLY_COVERED;
+    }
+  }
+
+  return ::mediapipe::OkStatus();
+}
+
+void FrameCropRegionComputer::UpdateCropRegionScore(
+    const KeyFrameCropOptions::ScoreAggregationType score_aggregation_type,
+    const float feature_score, const bool is_required,
+    float* crop_region_score) {
+  if (feature_score < 0.0) {
+    LOG(WARNING) << "Ignoring negative score";
+    return;
+  }
+
+  switch (score_aggregation_type) {
+    case KeyFrameCropOptions::MAXIMUM: {
+      *crop_region_score = std::max(feature_score, *crop_region_score);
+      break;
+    }
+    case KeyFrameCropOptions::SUM_REQUIRED: {
+      if (is_required) {
+        *crop_region_score += feature_score;
+      }
+      break;
+    }
+    case KeyFrameCropOptions::SUM_ALL: {
+      *crop_region_score += feature_score;
+      break;
+    }
+    case KeyFrameCropOptions::CONSTANT: {
+      *crop_region_score = 1.0;
+      break;
+    }
+    default: {
+      LOG(WARNING) << "Unknown CropRegionScoreType " << score_aggregation_type;
+      break;
+    }
+  }
+}
+
+::mediapipe::Status FrameCropRegionComputer::ComputeFrameCropRegion(
+    const KeyFrameInfo& frame_info, KeyFrameCropResult* crop_result) const {
+  RET_CHECK(crop_result != nullptr) << "KeyFrameCropResult is null.";
+
+  // Sorts required and non-required regions.
+  std::vector<SalientRegion> required_regions, non_required_regions;
+  const auto sort_status = SortDetections(
+      frame_info.detections(), &required_regions, &non_required_regions);
+  MP_RETURN_IF_ERROR(sort_status);
+
+  int target_width = options_.target_width();
+  int target_height = options_.target_height();
+  auto* region = crop_result->mutable_region();
+  RET_CHECK(region != nullptr) << "Crop region is null.";
+
+  bool crop_region_is_empty = true;
+  float crop_region_score = 0.0;
+
+  // Gets union of all required regions.
+  for (int i = 0; i < required_regions.size(); ++i) {
+    const Rect& required_region = required_regions[i].location();
+    if (crop_region_is_empty) {
+      *region = required_region;
+      crop_region_is_empty = false;
+    } else {
+      RectUnion(required_region, region);
+    }
+    UpdateCropRegionScore(options_.score_aggregation_type(),
+                          required_regions[i].score(), true,
+                          &crop_region_score);
+  }
+  crop_result->set_required_region_is_empty(crop_region_is_empty);
+  if (!crop_region_is_empty) {
+    *crop_result->mutable_required_region() = *region;
+    crop_result->set_are_required_regions_covered_in_target_size(
+        region->width() <= target_width && region->height() <= target_height);
+    target_width = std::max(target_width, region->width());
+    target_height = std::max(target_height, region->height());
+  } else {
+    crop_result->set_are_required_regions_covered_in_target_size(true);
+  }
+
+  // Tries to fit non-required regions.
+  int num_covered = 0;
+  for (int i = 0; i < non_required_regions.size(); ++i) {
+    const Rect& non_required_region = non_required_regions[i].location();
+    CoverType cover_type = NOT_COVERED;
+    if (crop_region_is_empty) {
+      // If the crop region is empty, tries to expand an empty base region
+      // at the center of this region to include itself.
+      region->set_x(non_required_region.x() + non_required_region.width() / 2);
+      region->set_y(non_required_region.y() + non_required_region.height() / 2);
+      region->set_width(0);
+      region->set_height(0);
+      MP_RETURN_IF_ERROR(ExpandRectUnderConstraints(non_required_region,
+                                                    target_width, target_height,
+                                                    region, &cover_type));
+      if (cover_type != NOT_COVERED) {
+        crop_region_is_empty = false;
+      }
+    } else {
+      // Otherwise tries to expand the crop region to cover the non-required
+      // region under target size constraint.
+      MP_RETURN_IF_ERROR(ExpandRectUnderConstraints(non_required_region,
+                                                    target_width, target_height,
+                                                    region, &cover_type));
+    }
+
+    // Updates number of covered non-required regions and score.
+    if (cover_type == FULLY_COVERED) {
+      num_covered++;
+      UpdateCropRegionScore(options_.score_aggregation_type(),
+                            non_required_regions[i].score(), false,
+                            &crop_region_score);
+    }
+  }
+
+  const float fraction_covered =
+      non_required_regions.empty()
+          ? 0.0
+          : static_cast<float>(num_covered) / non_required_regions.size();
+  crop_result->set_fraction_non_required_covered(fraction_covered);
+
+  crop_result->set_region_is_empty(crop_region_is_empty);
+  crop_result->set_region_score(crop_region_score);
+  return ::mediapipe::OkStatus();
+}
+
+}  // namespace autoflip
+}  // namespace mediapipe

mediapipe/examples/desktop/autoflip/quality/frame_crop_region_computer.h

@@ -0,0 +1,110 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_FRAME_CROP_REGION_COMPUTER_H_
+#define MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_FRAME_CROP_REGION_COMPUTER_H_
+
+#include <vector>
+
+#include "mediapipe/examples/desktop/autoflip/autoflip_messages.pb.h"
+#include "mediapipe/examples/desktop/autoflip/quality/cropping.pb.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+namespace autoflip {
+
+// This class computes per-frame crop regions based on crop frame options.
+// It aggregates required regions and then tries to fit in non-required regions
+// with best effort. It does not make use of static features.
+class FrameCropRegionComputer {
+ public:
+  FrameCropRegionComputer() = delete;
+
+  explicit FrameCropRegionComputer(
+      const KeyFrameCropOptions& crop_frame_options)
+      : options_(crop_frame_options) {}
+
+  ~FrameCropRegionComputer() {}
+
+  // Computes the crop region for the key frame using the crop options. The crop
+  // region covers all the required regions, and attempts to cover the
+  // non-required regions with best effort. Note: this function does not
+  // consider static features, and simply tries to fit the detected features
+  // within the target frame size. The score of the crop region is aggregated
+  // from individual feature scores given the score aggregation type.
+  ::mediapipe::Status ComputeFrameCropRegion(
+      const KeyFrameInfo& frame_info, KeyFrameCropResult* crop_result) const;
+
+ protected:
+  // A segment is a 1-d object defined by its left and right point.
+  using LeftPoint = int;
+  using RightPoint = int;
+  using Segment = std::pair<LeftPoint, RightPoint>;
+  // How much a segment is covered in the combined segment.
+  enum CoverType {
+    FULLY_COVERED = 1,
+    PARTIALLY_COVERED = 2,
+    NOT_COVERED = 3,
+  };
+  // Expands a base segment to cover a segment to be added given maximum length
+  // constraint. The operation is best-effort. The resulting enlarged segment is
+  // set in the returned combined segment. Returns a CoverType to indicate the
+  // coverage of the segment to be added in the combined segment.
+  // There are 3 cases:
+  //   case 1: the length of the union of the two segments is not larger than
+  //           the maximum length.
+  //           In this case the combined segment is simply the union, and cover
+  //           type is FULLY_COVERED.
+  //   case 2: the union of the two segments exceeds the maximum length, but the
+  //           union of the base segment and required minimum centered fraction
+  //           of the new segment fits in the maximum length.
+  //           In this case the combined segment is this latter union, and cover
+  //           type is PARTIALLY_COVERED.
+  //   case 3: the union of the base segment and required minimum centered
+  //           fraction of the new segment exceeds the maximum length.
+  //           In this case the combined segment is the base segment, and cover
+  //           type is NOT_COVERED.
+  ::mediapipe::Status ExpandSegmentUnderConstraint(
+      const Segment& segment_to_add, const Segment& base_segment,
+      const int max_length, Segment* combined_segment,
+      CoverType* cover_type) const;
+
+  // Expands a base rectangle to cover a new rectangle to be added under width
+  // and height constraints. The operation is best-effort. It considers
+  // horizontal and vertical directions separately, using the
+  // ExpandSegmentUnderConstraint function for each direction. The cover type is
+  // FULLY_COVERED if the new rectangle is fully covered in both directions,
+  // PARTIALLY_COVERED if it is at least partially covered in both directions,
+  // and NOT_COVERED if it is not covered in either direction.
+  ::mediapipe::Status ExpandRectUnderConstraints(const Rect& rect_to_add,
+                                                 const int max_width,
+                                                 const int max_height,
+                                                 Rect* base_rect,
+                                                 CoverType* cover_type) const;
+
+  // Updates crop region score given current feature score, whether the feature
+  // is required, and the score aggregation type. Ignores negative scores.
+  static void UpdateCropRegionScore(
+      const KeyFrameCropOptions::ScoreAggregationType score_aggregation_type,
+      const float feature_score, const bool is_required,
+      float* crop_region_score);
+
+ private:
+  // Crop frame options.
+  KeyFrameCropOptions options_;
+};
+}  // namespace autoflip
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_FRAME_CROP_REGION_COMPUTER_H_

mediapipe/examples/desktop/autoflip/quality/frame_crop_region_computer_test.cc

@@ -0,0 +1,579 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/examples/desktop/autoflip/quality/frame_crop_region_computer.h"
+
+#include <memory>
+
+#include "absl/memory/memory.h"
+#include "mediapipe/examples/desktop/autoflip/autoflip_messages.pb.h"
+#include "mediapipe/examples/desktop/autoflip/quality/cropping.pb.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/status_matchers.h"
+
+namespace mediapipe {
+namespace autoflip {
+
+using ::testing::HasSubstr;
+
+const int kSegmentMaxLength = 10;
+const int kTargetWidth = 500;
+const int kTargetHeight = 1000;
+
+// Makes a rectangle given the corner (x, y) and the size (width, height).
+Rect MakeRect(const int x, const int y, const int width, const int height) {
+  Rect rect;
+  rect.set_x(x);
+  rect.set_y(y);
+  rect.set_width(width);
+  rect.set_height(height);
+  return rect;
+}
+
+// Adds a detection to the key frame info given its location, whether it is
+// required, and its score. The score is default to 1.0.
+void AddDetection(const Rect& rect, const bool is_required,
+                  KeyFrameInfo* key_frame_info, const float score = 1.0) {
+  auto* detection = key_frame_info->mutable_detections()->add_detections();
+  *(detection->mutable_location()) = rect;
+  detection->set_score(score);
+  detection->set_is_required(is_required);
+}
+
+// Makes key frame crop options given target width and height.
+KeyFrameCropOptions MakeKeyFrameCropOptions(const int target_width,
+                                            const int target_height) {
+  KeyFrameCropOptions options;
+  options.set_target_width(target_width);
+  options.set_target_height(target_height);
+  return options;
+}
+
+// Checks whether rectangle a is inside rectangle b.
+bool CheckRectIsInside(const Rect& rect_a, const Rect& rect_b) {
+  return (rect_b.x() <= rect_a.x() && rect_b.y() <= rect_a.y() &&
+          rect_b.x() + rect_b.width() >= rect_a.x() + rect_a.width() &&
+          rect_b.y() + rect_b.height() >= rect_a.y() + rect_a.height());
+}
+
+// Checks whether two rectangles are equal.
+bool CheckRectsEqual(const Rect& rect1, const Rect& rect2) {
+  return (rect1.x() == rect2.x() && rect1.y() == rect2.y() &&
+          rect1.width() == rect2.width() && rect1.height() == rect2.height());
+}
+
+// Checks whether two rectangles have non-zero overlapping area.
+bool CheckRectsOverlap(const Rect& rect1, const Rect& rect2) {
+  const int x1_left = rect1.x(), x1_right = rect1.x() + rect1.width();
+  const int y1_top = rect1.y(), y1_bottom = rect1.y() + rect1.height();
+  const int x2_left = rect2.x(), x2_right = rect2.x() + rect2.width();
+  const int y2_top = rect2.y(), y2_bottom = rect2.y() + rect2.height();
+  const int x_left = std::max(x1_left, x2_left);
+  const int x_right = std::min(x1_right, x2_right);
+  const int y_top = std::max(y1_top, y2_top);
+  const int y_bottom = std::min(y1_bottom, y2_bottom);
+  return (x_right > x_left && y_bottom > y_top);
+}
+
+// Checks that all the required regions in the detections in KeyFrameInfo are
+// covered in the KeyFrameCropResult.
+void CheckRequiredRegionsAreCovered(const KeyFrameInfo& key_frame_info,
+                                    const KeyFrameCropResult& result) {
+  bool has_required = false;
+  for (int i = 0; i < key_frame_info.detections().detections_size(); ++i) {
+    const auto& detection = key_frame_info.detections().detections(i);
+    if (detection.is_required()) {
+      has_required = true;
+      EXPECT_TRUE(
+          CheckRectIsInside(detection.location(), result.required_region()));
+    }
+  }
+  EXPECT_EQ(has_required, !result.required_region_is_empty());
+  if (has_required) {
+    EXPECT_FALSE(result.region_is_empty());
+    EXPECT_TRUE(CheckRectIsInside(result.required_region(), result.region()));
+  }
+}
+
+// Testable class that can access protected types and methods in the class.
+class TestableFrameCropRegionComputer : public FrameCropRegionComputer {
+ public:
+  explicit TestableFrameCropRegionComputer(const KeyFrameCropOptions& options)
+      : FrameCropRegionComputer(options) {}
+  using FrameCropRegionComputer::CoverType;
+  using FrameCropRegionComputer::ExpandRectUnderConstraints;
+  using FrameCropRegionComputer::ExpandSegmentUnderConstraint;
+  using FrameCropRegionComputer::FULLY_COVERED;
+  using FrameCropRegionComputer::LeftPoint;  // int
+  using FrameCropRegionComputer::NOT_COVERED;
+  using FrameCropRegionComputer::PARTIALLY_COVERED;
+  using FrameCropRegionComputer::RightPoint;  // int
+  using FrameCropRegionComputer::Segment;     // std::pair<int, int>
+  using FrameCropRegionComputer::UpdateCropRegionScore;
+
+  // Makes a segment from two endpoints.
+  static Segment MakeSegment(const LeftPoint left, const RightPoint right) {
+    return std::make_pair(left, right);
+  }
+
+  // Checks that two segments are equal.
+  static bool CheckSegmentsEqual(const Segment& segment1,
+                                 const Segment& segment2) {
+    return (segment1.first == segment2.first &&
+            segment1.second == segment2.second);
+  }
+};
+using TestClass = TestableFrameCropRegionComputer;
+
+// Returns an instance of the testable class given
+// non_required_region_min_coverage_fraction.
+std::unique_ptr<TestClass> GetTestableClass(
+    const float non_required_region_min_coverage_fraction = 0.5) {
+  KeyFrameCropOptions options;
+  options.set_non_required_region_min_coverage_fraction(
+      non_required_region_min_coverage_fraction);
+  auto test_class = absl::make_unique<TestClass>(options);
+  return test_class;
+}
+
+// Checks that ExpandSegmentUnderConstraint checks output pointers are not null.
+TEST(FrameCropRegionComputerTest, ExpandSegmentUnderConstraintCheckNull) {
+  auto test_class = GetTestableClass();
+  TestClass::CoverType cover_type;
+  TestClass::Segment base_segment = TestClass::MakeSegment(10, 15);
+  TestClass::Segment segment_to_add = TestClass::MakeSegment(5, 8);
+  TestClass::Segment combined_segment;
+  // Combined segment is null.
+  auto status = test_class->ExpandSegmentUnderConstraint(
+      segment_to_add, base_segment, kSegmentMaxLength, nullptr, &cover_type);
+  EXPECT_FALSE(status.ok());
+  EXPECT_THAT(status.ToString(), HasSubstr("Combined segment is null."));
+  // Cover type is null.
+  status = test_class->ExpandSegmentUnderConstraint(
+      segment_to_add, base_segment, kSegmentMaxLength, &combined_segment,
+      nullptr);
+  EXPECT_FALSE(status.ok());
+  EXPECT_THAT(status.ToString(), HasSubstr("Cover type is null."));
+}
+
+// Checks that ExpandSegmentUnderConstraint checks input segments are valid.
+TEST(FrameCropRegionComputerTest, ExpandSegmentUnderConstraintCheckValid) {
+  auto test_class = GetTestableClass();
+  TestClass::CoverType cover_type;
+  TestClass::Segment combined_segment;
+
+  // Invalid base segment.
+  TestClass::Segment base_segment = TestClass::MakeSegment(15, 10);
+  TestClass::Segment segment_to_add = TestClass::MakeSegment(5, 8);
+  auto status = test_class->ExpandSegmentUnderConstraint(
+      segment_to_add, base_segment, kSegmentMaxLength, &combined_segment,
+      &cover_type);
+  EXPECT_FALSE(status.ok());
+  EXPECT_THAT(status.ToString(), HasSubstr("Invalid base segment."));
+
+  // Invalid segment to add.
+  base_segment = TestClass::MakeSegment(10, 15);
+  segment_to_add = TestClass::MakeSegment(8, 5);
+  status = test_class->ExpandSegmentUnderConstraint(
+      segment_to_add, base_segment, kSegmentMaxLength, &combined_segment,
+      &cover_type);
+  EXPECT_FALSE(status.ok());
+  EXPECT_THAT(status.ToString(), HasSubstr("Invalid segment to add."));
+
+  // Base segment exceeds max length.
+  base_segment = TestClass::MakeSegment(10, 100);
+  segment_to_add = TestClass::MakeSegment(5, 8);
+  status = test_class->ExpandSegmentUnderConstraint(
+      segment_to_add, base_segment, kSegmentMaxLength, &combined_segment,
+      &cover_type);
+  EXPECT_FALSE(status.ok());
+  EXPECT_THAT(status.ToString(),
+              HasSubstr("Base segment length exceeds max length."));
+}
+
+// Checks that ExpandSegmentUnderConstraint handles case 1 properly: the length
+// of the union of the two segments is not larger than the maximum length.
+TEST(FrameCropRegionComputerTest, ExpandSegmentUnderConstraintCase1) {
+  auto test_class = GetTestableClass();
+  TestClass::Segment combined_segment;
+  TestClass::CoverType cover_type;
+  TestClass::Segment base_segment = TestClass::MakeSegment(5, 10);
+  TestClass::Segment segment_to_add = TestClass::MakeSegment(3, 8);
+  MP_EXPECT_OK(test_class->ExpandSegmentUnderConstraint(
+      segment_to_add, base_segment, kSegmentMaxLength, &combined_segment,
+      &cover_type));
+  EXPECT_EQ(cover_type, TestClass::FULLY_COVERED);
+  EXPECT_TRUE(TestClass::CheckSegmentsEqual(combined_segment,
+                                            TestClass::MakeSegment(3, 10)));
+}
+
+// Checks that ExpandSegmentUnderConstraint handles case 2 properly: the union
+// of the two segments exceeds the maximum length, but the union of the base
+// segment with the minimum coverage fraction of the new segment is within the
+// maximum length.
+TEST(FrameCropRegionComputerTest, ExpandSegmentUnderConstraintCase2) {
+  TestClass::Segment combined_segment;
+  TestClass::CoverType cover_type;
+  TestClass::Segment base_segment = TestClass::MakeSegment(4, 8);
+  TestClass::Segment segment_to_add = TestClass::MakeSegment(0, 16);
+  auto test_class = GetTestableClass();
+  MP_EXPECT_OK(test_class->ExpandSegmentUnderConstraint(
+      segment_to_add, base_segment, kSegmentMaxLength, &combined_segment,
+      &cover_type));
+  EXPECT_EQ(cover_type, TestClass::PARTIALLY_COVERED);
+  EXPECT_TRUE(TestClass::CheckSegmentsEqual(combined_segment,
+                                            TestClass::MakeSegment(4, 12)));
+}
+
+// Checks that ExpandSegmentUnderConstraint handles case 3 properly: the union
+// of the base segment with the minimum coverage fraction of the new segment
+// exceeds the maximum length.
+TEST(FrameCropRegionComputerTest, ExpandSegmentUnderConstraintCase3) {
+  TestClass::Segment combined_segment;
+  TestClass::CoverType cover_type;
+  auto test_class = GetTestableClass();
+  TestClass::Segment base_segment = TestClass::MakeSegment(6, 14);
+  TestClass::Segment segment_to_add = TestClass::MakeSegment(0, 4);
+  MP_EXPECT_OK(test_class->ExpandSegmentUnderConstraint(
+      segment_to_add, base_segment, kSegmentMaxLength, &combined_segment,
+      &cover_type));
+  EXPECT_EQ(cover_type, TestClass::NOT_COVERED);
+  EXPECT_TRUE(TestClass::CheckSegmentsEqual(combined_segment, base_segment));
+}
+
+// Checks that ExpandRectUnderConstraints checks output pointers are not null.
+TEST(FrameCropRegionComputerTest, ExpandRectUnderConstraintsChecksNotNull) {
+  auto test_class = GetTestableClass();
+  TestClass::CoverType cover_type;
+  Rect base_rect, rect_to_add;
+  // Base rect is null.
+  auto status = test_class->ExpandRectUnderConstraints(
+      rect_to_add, kTargetWidth, kTargetHeight, nullptr, &cover_type);
+  EXPECT_FALSE(status.ok());
+  EXPECT_THAT(status.ToString(), HasSubstr("Base rect is null."));
+  // Cover type is null.
+  status = test_class->ExpandRectUnderConstraints(
+      rect_to_add, kTargetWidth, kTargetHeight, &base_rect, nullptr);
+  EXPECT_FALSE(status.ok());
+  EXPECT_THAT(status.ToString(), HasSubstr("Cover type is null."));
+}
+
+// Checks that ExpandRectUnderConstraints checks base rect is valid.
+TEST(FrameCropRegionComputerTest, ExpandRectUnderConstraintsChecksBaseValid) {
+  auto test_class = GetTestableClass();
+  TestClass::CoverType cover_type;
+  Rect base_rect = MakeRect(0, 0, 2 * kTargetWidth, 2 * kTargetHeight);
+  Rect rect_to_add;
+  const auto status = test_class->ExpandRectUnderConstraints(
+      rect_to_add, kTargetWidth, kTargetHeight, &base_rect, &cover_type);
+  EXPECT_FALSE(status.ok());
+  EXPECT_THAT(status.ToString(),
+              HasSubstr("Base rect already exceeds target size."));
+}
+
+// Checks that ExpandRectUnderConstraints properly handles the case where the
+// rectangle to be added can be fully covered.
+TEST(FrameCropRegionComputerTest, ExpandRectUnderConstraintsFullyCovered) {
+  auto test_class = GetTestableClass();
+  TestClass::CoverType cover_type;
+  Rect base_rect = MakeRect(0, 0, 50, 50);
+  Rect rect_to_add = MakeRect(30, 30, 30, 30);
+  MP_EXPECT_OK(test_class->ExpandRectUnderConstraints(
+      rect_to_add, kTargetWidth, kTargetHeight, &base_rect, &cover_type));
+  EXPECT_EQ(cover_type, TestClass::FULLY_COVERED);
+  EXPECT_TRUE(CheckRectsEqual(base_rect, MakeRect(0, 0, 60, 60)));
+}
+
+// Checks that ExpandRectUnderConstraints properly handles the case where the
+// rectangle to be added can be partially covered.
+TEST(FrameCropRegionComputerTest, ExpandRectUnderConstraintsPartiallyCovered) {
+  auto test_class = GetTestableClass();
+  TestClass::CoverType cover_type;
+  // Rectangle to be added can be partially covered in both both dimensions.
+  Rect base_rect = MakeRect(0, 0, 500, 500);
+  Rect rect_to_add = MakeRect(0, 300, 600, 900);
+  MP_EXPECT_OK(test_class->ExpandRectUnderConstraints(
+      rect_to_add, kTargetWidth, kTargetHeight, &base_rect, &cover_type));
+  EXPECT_EQ(cover_type, TestClass::PARTIALLY_COVERED);
+  EXPECT_TRUE(CheckRectsEqual(base_rect, MakeRect(0, 0, 500, 975)));
+
+  // Rectangle to be added can be fully covered in one dimension and partially
+  // covered in the other dimension.
+  base_rect = MakeRect(0, 0, 400, 500);
+  rect_to_add = MakeRect(100, 300, 400, 900);
+  MP_EXPECT_OK(test_class->ExpandRectUnderConstraints(
+      rect_to_add, kTargetWidth, kTargetHeight, &base_rect, &cover_type));
+  EXPECT_EQ(cover_type, TestClass::PARTIALLY_COVERED);
+  EXPECT_TRUE(CheckRectsEqual(base_rect, MakeRect(0, 0, 500, 975)));
+}
+
+// Checks that ExpandRectUnderConstraints properly handles the case where the
+// rectangle to be added cannot be covered.
+TEST(FrameCropRegionComputerTest, ExpandRectUnderConstraintsNotCovered) {
+  TestClass::CoverType cover_type;
+  auto test_class = GetTestableClass();
+  Rect base_rect = MakeRect(0, 0, 500, 500);
+  Rect rect_to_add = MakeRect(550, 300, 100, 900);
+  MP_EXPECT_OK(test_class->ExpandRectUnderConstraints(
+      rect_to_add, kTargetWidth, kTargetHeight, &base_rect, &cover_type));
+  EXPECT_EQ(cover_type, TestClass::NOT_COVERED);  // no overlap in x dimension
+  EXPECT_TRUE(CheckRectsEqual(base_rect, MakeRect(0, 0, 500, 500)));
+}
+
+// Checks that ComputeFrameCropRegion handles the case of empty detections.
+TEST(FrameCropRegionComputerTest, HandlesEmptyDetections) {
+  const auto options = MakeKeyFrameCropOptions(kTargetWidth, kTargetHeight);
+  FrameCropRegionComputer computer(options);
+  KeyFrameInfo key_frame_info;
+  KeyFrameCropResult crop_result;
+  MP_EXPECT_OK(computer.ComputeFrameCropRegion(key_frame_info, &crop_result));
+  EXPECT_TRUE(crop_result.region_is_empty());
+}
+
+// Checks that ComputeFrameCropRegion covers required regions when their union
+// is within target size.
+TEST(FrameCropRegionComputerTest, CoversRequiredWithinTargetSize) {
+  const auto options = MakeKeyFrameCropOptions(kTargetWidth, kTargetHeight);
+  FrameCropRegionComputer computer(options);
+  KeyFrameInfo key_frame_info;
+  AddDetection(MakeRect(100, 100, 100, 200), true, &key_frame_info);
+  AddDetection(MakeRect(200, 400, 300, 500), true, &key_frame_info);
+  KeyFrameCropResult crop_result;
+  MP_EXPECT_OK(computer.ComputeFrameCropRegion(key_frame_info, &crop_result));
+  CheckRequiredRegionsAreCovered(key_frame_info, crop_result);
+  EXPECT_TRUE(CheckRectsEqual(MakeRect(100, 100, 400, 800),
+                              crop_result.required_region()));
+  EXPECT_TRUE(
+      CheckRectsEqual(crop_result.region(), crop_result.required_region()));
+  EXPECT_TRUE(crop_result.are_required_regions_covered_in_target_size());
+}
+
+// Checks that ComputeFrameCropRegion covers required regions when their union
+// exceeds target size.
+TEST(FrameCropRegionComputerTest, CoversRequiredExceedingTargetSize) {
+  const auto options = MakeKeyFrameCropOptions(kTargetWidth, kTargetHeight);
+  FrameCropRegionComputer computer(options);
+  KeyFrameInfo key_frame_info;
+  AddDetection(MakeRect(0, 0, 100, 500), true, &key_frame_info);
+  AddDetection(MakeRect(200, 400, 500, 500), true, &key_frame_info);
+  KeyFrameCropResult crop_result;
+  MP_EXPECT_OK(computer.ComputeFrameCropRegion(key_frame_info, &crop_result));
+  CheckRequiredRegionsAreCovered(key_frame_info, crop_result);
+  EXPECT_TRUE(CheckRectsEqual(MakeRect(0, 0, 700, 900), crop_result.region()));
+  EXPECT_TRUE(
+      CheckRectsEqual(crop_result.region(), crop_result.required_region()));
+  EXPECT_FALSE(crop_result.are_required_regions_covered_in_target_size());
+}
+
+// Checks that ComputeFrameCropRegion handles the case of only non-required
+// regions and the region fits in the target size.
+TEST(FrameCropRegionComputerTest,
+     HandlesOnlyNonRequiedRegionsInsideTargetSize) {
+  const auto options = MakeKeyFrameCropOptions(kTargetWidth, kTargetHeight);
+  FrameCropRegionComputer computer(options);
+  KeyFrameInfo key_frame_info;
+  AddDetection(MakeRect(300, 600, 100, 100), false, &key_frame_info);
+  KeyFrameCropResult crop_result;
+  MP_EXPECT_OK(computer.ComputeFrameCropRegion(key_frame_info, &crop_result));
+  EXPECT_TRUE(crop_result.required_region_is_empty());
+  EXPECT_FALSE(crop_result.region_is_empty());
+  EXPECT_TRUE(
+      CheckRectsEqual(key_frame_info.detections().detections(0).location(),
+                      crop_result.region()));
+}
+
+// Checks that ComputeFrameCropRegion handles the case of only non-required
+// regions and the region exceeds the target size.
+TEST(FrameCropRegionComputerTest,
+     HandlesOnlyNonRequiedRegionsExceedingTargetSize) {
+  const auto options = MakeKeyFrameCropOptions(kTargetWidth, kTargetHeight);
+  FrameCropRegionComputer computer(options);
+  KeyFrameInfo key_frame_info;
+  AddDetection(MakeRect(300, 600, 700, 100), false, &key_frame_info);
+  KeyFrameCropResult crop_result;
+  MP_EXPECT_OK(computer.ComputeFrameCropRegion(key_frame_info, &crop_result));
+  EXPECT_TRUE(crop_result.required_region_is_empty());
+  EXPECT_FALSE(crop_result.region_is_empty());
+  EXPECT_TRUE(
+      CheckRectsEqual(MakeRect(475, 600, 350, 100), crop_result.region()));
+  EXPECT_EQ(crop_result.fraction_non_required_covered(), 0.0);
+  EXPECT_TRUE(
+      CheckRectIsInside(crop_result.region(),
+                        key_frame_info.detections().detections(0).location()));
+}
+
+// Checks that ComputeFrameCropRegion covers non-required regions when their
+// union fits within target size.
+TEST(FrameCropRegionComputerTest, CoversNonRequiredInsideTargetSize) {
+  const auto options = MakeKeyFrameCropOptions(kTargetWidth, kTargetHeight);
+  FrameCropRegionComputer computer(options);
+  KeyFrameInfo key_frame_info;
+  AddDetection(MakeRect(0, 0, 100, 500), true, &key_frame_info);
+  AddDetection(MakeRect(300, 600, 100, 100), false, &key_frame_info);
+  KeyFrameCropResult crop_result;
+  MP_EXPECT_OK(computer.ComputeFrameCropRegion(key_frame_info, &crop_result));
+  CheckRequiredRegionsAreCovered(key_frame_info, crop_result);
+  EXPECT_TRUE(CheckRectsEqual(MakeRect(0, 0, 400, 700), crop_result.region()));
+  EXPECT_TRUE(crop_result.are_required_regions_covered_in_target_size());
+  EXPECT_EQ(crop_result.fraction_non_required_covered(), 1.0);
+  for (int i = 0; i < key_frame_info.detections().detections_size(); ++i) {
+    EXPECT_TRUE(
+        CheckRectIsInside(key_frame_info.detections().detections(i).location(),
+                          crop_result.region()));
+  }
+}
+
+// Checks that ComputeFrameCropRegion does not cover non-required regions that
+// are outside the target size.
+TEST(FrameCropRegionComputerTest, DoesNotCoverNonRequiredExceedingTargetSize) {
+  const auto options = MakeKeyFrameCropOptions(kTargetWidth, kTargetHeight);
+  FrameCropRegionComputer computer(options);
+  KeyFrameInfo key_frame_info;
+  AddDetection(MakeRect(0, 0, 500, 1000), true, &key_frame_info);
+  AddDetection(MakeRect(500, 0, 100, 100), false, &key_frame_info);
+  KeyFrameCropResult crop_result;
+  MP_EXPECT_OK(computer.ComputeFrameCropRegion(key_frame_info, &crop_result));
+  CheckRequiredRegionsAreCovered(key_frame_info, crop_result);
+  EXPECT_TRUE(CheckRectsEqual(MakeRect(0, 0, 500, 1000), crop_result.region()));
+  EXPECT_TRUE(crop_result.are_required_regions_covered_in_target_size());
+  EXPECT_EQ(crop_result.fraction_non_required_covered(), 0.0);
+  EXPECT_FALSE(
+      CheckRectIsInside(key_frame_info.detections().detections(1).location(),
+                        crop_result.region()));
+}
+
+// Checks that ComputeFrameCropRegion partially covers non-required regions that
+// can partially fit in the target size.
+TEST(FrameCropRegionComputerTest,
+     PartiallyCoversNonRequiredContainingTargetSize) {
+  const auto options = MakeKeyFrameCropOptions(kTargetWidth, kTargetHeight);
+  FrameCropRegionComputer computer(options);
+  KeyFrameInfo key_frame_info;
+  AddDetection(MakeRect(100, 0, 350, 1000), true, &key_frame_info);
+  AddDetection(MakeRect(0, 0, 650, 100), false, &key_frame_info);
+  KeyFrameCropResult crop_result;
+  MP_EXPECT_OK(computer.ComputeFrameCropRegion(key_frame_info, &crop_result));
+  CheckRequiredRegionsAreCovered(key_frame_info, crop_result);
+  EXPECT_TRUE(
+      CheckRectsEqual(MakeRect(100, 0, 387, 1000), crop_result.region()));
+  EXPECT_TRUE(crop_result.are_required_regions_covered_in_target_size());
+  EXPECT_EQ(crop_result.fraction_non_required_covered(), 0.0);
+  EXPECT_TRUE(
+      CheckRectsOverlap(key_frame_info.detections().detections(1).location(),
+                        crop_result.region()));
+}
+
+// Checks that ComputeFrameCropRegion covers non-required regions when the
+// required regions exceed target size.
+TEST(FrameCropRegionComputerTest,
+     CoversNonRequiredWhenRequiredExceedsTargetSize) {
+  const auto options = MakeKeyFrameCropOptions(kTargetWidth, kTargetHeight);
+  FrameCropRegionComputer computer(options);
+  KeyFrameInfo key_frame_info;
+  AddDetection(MakeRect(0, 0, 600, 1000), true, &key_frame_info);
+  AddDetection(MakeRect(450, 0, 100, 100), false, &key_frame_info);
+  KeyFrameCropResult crop_result;
+  MP_EXPECT_OK(computer.ComputeFrameCropRegion(key_frame_info, &crop_result));
+  CheckRequiredRegionsAreCovered(key_frame_info, crop_result);
+  EXPECT_TRUE(CheckRectsEqual(MakeRect(0, 0, 600, 1000), crop_result.region()));
+  EXPECT_FALSE(crop_result.are_required_regions_covered_in_target_size());
+  EXPECT_EQ(crop_result.fraction_non_required_covered(), 1.0);
+  for (int i = 0; i < key_frame_info.detections().detections_size(); ++i) {
+    EXPECT_TRUE(
+        CheckRectIsInside(key_frame_info.detections().detections(i).location(),
+                          crop_result.region()));
+  }
+}
+
+// Checks that ComputeFrameCropRegion does not extend the crop region when
+// the non-required region is too far.
+TEST(FrameCropRegionComputerTest,
+     DoesNotExtendRegionWhenNonRequiredRegionIsTooFar) {
+  const auto options = MakeKeyFrameCropOptions(kTargetWidth, kTargetHeight);
+  FrameCropRegionComputer computer(options);
+  KeyFrameInfo key_frame_info;
+  AddDetection(MakeRect(0, 0, 400, 400), true, &key_frame_info);
+  AddDetection(MakeRect(600, 0, 100, 100), false, &key_frame_info);
+  KeyFrameCropResult crop_result;
+  MP_EXPECT_OK(computer.ComputeFrameCropRegion(key_frame_info, &crop_result));
+  CheckRequiredRegionsAreCovered(key_frame_info, crop_result);
+  EXPECT_TRUE(CheckRectsEqual(MakeRect(0, 0, 400, 400), crop_result.region()));
+  EXPECT_TRUE(crop_result.are_required_regions_covered_in_target_size());
+  EXPECT_EQ(crop_result.fraction_non_required_covered(), 0.0);
+  EXPECT_FALSE(
+      CheckRectsOverlap(key_frame_info.detections().detections(1).location(),
+                        crop_result.region()));
+}
+
+// Checks that ComputeFrameCropRegion computes the score correctly when the
+// aggregation type is maximum.
+TEST(FrameCropRegionComputerTest, ComputesScoreWhenAggregationIsMaximum) {
+  auto options = MakeKeyFrameCropOptions(kTargetWidth, kTargetHeight);
+  options.set_score_aggregation_type(KeyFrameCropOptions::MAXIMUM);
+  FrameCropRegionComputer computer(options);
+  KeyFrameInfo key_frame_info;
+  AddDetection(MakeRect(0, 0, 400, 400), true, &key_frame_info, 0.1);
+  AddDetection(MakeRect(300, 300, 200, 500), true, &key_frame_info, 0.9);
+  KeyFrameCropResult crop_result;
+  MP_EXPECT_OK(computer.ComputeFrameCropRegion(key_frame_info, &crop_result));
+  EXPECT_FLOAT_EQ(crop_result.region_score(), 0.9f);
+}
+
+// Checks that ComputeFrameCropRegion computes the score correctly when the
+// aggregation type is sum required regions.
+TEST(FrameCropRegionComputerTest, ComputesScoreWhenAggregationIsSumRequired) {
+  auto options = MakeKeyFrameCropOptions(kTargetWidth, kTargetHeight);
+  options.set_score_aggregation_type(KeyFrameCropOptions::SUM_REQUIRED);
+  FrameCropRegionComputer computer(options);
+  KeyFrameInfo key_frame_info;
+  AddDetection(MakeRect(0, 0, 400, 400), true, &key_frame_info, 0.1);
+  AddDetection(MakeRect(300, 300, 200, 500), true, &key_frame_info, 0.9);
+  AddDetection(MakeRect(300, 300, 200, 500), false, &key_frame_info, 0.5);
+  KeyFrameCropResult crop_result;
+  MP_EXPECT_OK(computer.ComputeFrameCropRegion(key_frame_info, &crop_result));
+  EXPECT_FLOAT_EQ(crop_result.region_score(), 1.0f);
+}
+
+// Checks that ComputeFrameCropRegion computes the score correctly when the
+// aggregation type is sum all covered regions.
+TEST(FrameCropRegionComputerTest, ComputesScoreWhenAggregationIsSumAll) {
+  auto options = MakeKeyFrameCropOptions(kTargetWidth, kTargetHeight);
+  options.set_score_aggregation_type(KeyFrameCropOptions::SUM_ALL);
+  FrameCropRegionComputer computer(options);
+  KeyFrameInfo key_frame_info;
+  AddDetection(MakeRect(0, 0, 400, 400), true, &key_frame_info, 0.1);
+  AddDetection(MakeRect(300, 300, 200, 500), true, &key_frame_info, 0.9);
+  AddDetection(MakeRect(300, 300, 200, 500), false, &key_frame_info, 0.5);
+  KeyFrameCropResult crop_result;
+  MP_EXPECT_OK(computer.ComputeFrameCropRegion(key_frame_info, &crop_result));
+  EXPECT_FLOAT_EQ(crop_result.region_score(), 1.5f);
+}
+
+// Checks that ComputeFrameCropRegion computes the score correctly when the
+// aggregation type is constant.
+TEST(FrameCropRegionComputerTest, ComputesScoreWhenAggregationIsConstant) {
+  auto options = MakeKeyFrameCropOptions(kTargetWidth, kTargetHeight);
+  options.set_score_aggregation_type(KeyFrameCropOptions::CONSTANT);
+  FrameCropRegionComputer computer(options);
+  KeyFrameInfo key_frame_info;
+  AddDetection(MakeRect(0, 0, 400, 400), true, &key_frame_info, 0.1);
+  AddDetection(MakeRect(300, 300, 200, 500), true, &key_frame_info, 0.9);
+  AddDetection(MakeRect(300, 300, 200, 500), false, &key_frame_info, 0.5);
+  KeyFrameCropResult crop_result;
+  MP_EXPECT_OK(computer.ComputeFrameCropRegion(key_frame_info, &crop_result));
+  EXPECT_FLOAT_EQ(crop_result.region_score(), 1.0f);
+}
+}  // namespace autoflip
+}  // namespace mediapipe

mediapipe/examples/desktop/autoflip/quality/math_utils.h

@@ -0,0 +1,40 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_MATH_UTILS_H_
+#define MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_MATH_UTILS_H_
+
+class MathUtil {
+ public:
+  // Clamps value to the range [low, high].  Requires low <= high. Returns false
+  // if this check fails, otherwise returns true. Caller should first check the
+  // returned boolean.
+  template <typename T>  // T models LessThanComparable.
+  static bool Clamp(const T& low, const T& high, const T& value, T* result) {
+    // Prevents errors in ordering the arguments.
+    if (low > high) {
+      return false;
+    }
+    if (high < value) {
+      *result = high;
+    } else if (value < low) {
+      *result = low;
+    } else {
+      *result = value;
+    }
+    return true;
+  }
+};
+
+#endif  // MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_MATH_UTILS_H_

mediapipe/examples/desktop/autoflip/quality/padding_effect_generator.cc

@@ -0,0 +1,177 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/examples/desktop/autoflip/quality/padding_effect_generator.h"
+
+#include "mediapipe/framework/formats/image_frame.h"
+#include "mediapipe/framework/formats/image_frame_opencv.h"
+#include "mediapipe/framework/port/opencv_core_inc.h"
+#include "mediapipe/framework/port/opencv_imgproc_inc.h"
+#include "mediapipe/framework/port/ret_check.h"
+
+namespace mediapipe {
+namespace autoflip {
+
+PaddingEffectGenerator::PaddingEffectGenerator(const int input_width,
+                                               const int input_height,
+                                               const double target_aspect_ratio,
+                                               bool scale_to_multiple_of_two) {
+  target_aspect_ratio_ = target_aspect_ratio;
+  const double input_aspect_ratio =
+      static_cast<double>(input_width) / static_cast<double>(input_height);
+  input_width_ = input_width;
+  input_height_ = input_height;
+  is_vertical_padding_ = input_aspect_ratio > target_aspect_ratio;
+  output_width_ = is_vertical_padding_
+                      ? std::round(target_aspect_ratio * input_height)
+                      : input_width;
+  output_height_ = is_vertical_padding_
+                       ? input_height
+                       : std::round(input_width / target_aspect_ratio);
+  if (scale_to_multiple_of_two) {
+    output_width_ = output_width_ / 2 * 2;
+    output_height_ = output_height_ / 2 * 2;
+  }
+}
+
+::mediapipe::Status PaddingEffectGenerator::Process(
+    const ImageFrame& input_frame, const float background_contrast,
+    const int blur_cv_size, const float overlay_opacity,
+    ImageFrame* output_frame, const cv::Scalar* background_color_in_rgb) {
+  RET_CHECK_EQ(input_frame.Width(), input_width_);
+  RET_CHECK_EQ(input_frame.Height(), input_height_);
+  RET_CHECK(output_frame);
+
+  cv::Mat original_image = formats::MatView(&input_frame);
+  // This is the canvas that we are going to draw the padding effect on to.
+  cv::Mat canvas(output_height_, output_width_, original_image.type());
+
+  const int effective_input_width =
+      is_vertical_padding_ ? input_width_ : input_height_;
+  const int effective_input_height =
+      is_vertical_padding_ ? input_height_ : input_width_;
+  const int effective_output_width =
+      is_vertical_padding_ ? output_width_ : output_height_;
+  const int effective_output_height =
+      is_vertical_padding_ ? output_height_ : output_width_;
+
+  if (!is_vertical_padding_) {
+    original_image = original_image.t();
+    canvas = canvas.t();
+  }
+
+  const int foreground_height =
+      effective_input_height * effective_output_width / effective_input_width;
+  int x = -1, y = -1, width = -1, height = -1;
+
+  // The following steps does the padding operation, with several steps.
+  // #1, we prepare the background. If a solid background color is given, we use
+  //     it directly. Otherwise, we first crop a region of size "output_width_ *
+  //     output_height_" off of the original frame to become the background of
+  //     the final frame, and then we blur it and adjust contrast and opacity.
+  if (background_color_in_rgb != nullptr) {
+    canvas = *background_color_in_rgb;
+  } else {
+    // Copy the original image to the background.
+    x = 0.5 * (effective_input_width - effective_output_width);
+    y = 0;
+    width = effective_output_width;
+    height = effective_output_height;
+    cv::Rect crop_window_for_background(x, y, width, height);
+    original_image(crop_window_for_background).copyTo(canvas);
+
+    // Blur.
+    const int cv_size =
+        blur_cv_size % 2 == 1 ? blur_cv_size : (blur_cv_size + 1);
+    const cv::Size kernel(cv_size, cv_size);
+    // TODO: the larger the kernel size, the slower the blurring
+    // operation is. Consider running multiple sequential blurs with smaller
+    // sizes to simulate the effect of using a large size. This might be able to
+    // speed up the process.
+    x = 0;
+    width = effective_output_width;
+    const cv::Rect canvas_rect(0, 0, canvas.cols, canvas.rows);
+    // Blur the top region (above foreground).
+    y = 0;
+    height = (effective_output_height - foreground_height) / 2 + cv_size;
+    const cv::Rect top_blur_region =
+        cv::Rect(x, y, width, height) & canvas_rect;
+    if (top_blur_region.area() > 0) {
+      cv::Mat top_blurred = canvas(top_blur_region);
+      cv::GaussianBlur(top_blurred, top_blurred, kernel, 0, 0);
+    }
+    // Blur the bottom region (below foreground).
+    y = height + foreground_height - cv_size;
+    height = effective_output_height - y;
+    const cv::Rect bottom_blur_region =
+        cv::Rect(x, y, width, height) & canvas_rect;
+    if (bottom_blur_region.area() > 0) {
+      cv::Mat bottom_blurred = canvas(bottom_blur_region);
+      cv::GaussianBlur(bottom_blurred, bottom_blurred, kernel, 0, 0);
+    }
+
+    const float kEqualThreshold = 0.0001f;
+    // Background contrast adjustment.
+    if (std::abs(background_contrast - 1.0f) > kEqualThreshold) {
+      canvas *= background_contrast;
+    }
+
+    // Alpha blend a translucent black layer.
+    if (std::abs(overlay_opacity - 0.0f) > kEqualThreshold) {
+      cv::Mat overlay = cv::Mat::zeros(canvas.size(), canvas.type());
+      cv::addWeighted(overlay, overlay_opacity, canvas, 1 - overlay_opacity, 0,
+                      canvas);
+    }
+  }
+
+  // #2, we crop the entire region off of the original frame. This will become
+  //     the foreground in the final frame.
+  x = 0;
+  y = 0;
+  width = effective_input_width;
+  height = effective_input_height;
+
+  cv::Rect crop_window_for_foreground(x, y, width, height);
+
+  // #3, we specify a region of size computed as below in the final frame to
+  //     embed the foreground that we obtained in #2. The aspect ratio of
+  //     this region should be the same as the foreground, but with a
+  //     smaller size. Therefore, the height and width are derived using
+  //     the ratio of the sizes.
+  //     - embed size: output_width_ * height (to be computed)
+  //     - foreground: input_width * input_height
+  //
+  //     The location of this region is horizontally centralized in the
+  //     frame, and saturated in horizontal dimension.
+  x = 0;
+  y = (effective_output_height - foreground_height) / 2;
+  width = effective_output_width;
+  height = foreground_height;
+
+  cv::Rect region_to_embed_foreground(x, y, width, height);
+  cv::Mat dst = canvas(region_to_embed_foreground);
+  cv::resize(original_image(crop_window_for_foreground), dst, dst.size());
+
+  if (!is_vertical_padding_) {
+    canvas = canvas.t();
+  }
+
+  output_frame->CopyPixelData(input_frame.Format(), canvas.cols, canvas.rows,
+                              canvas.data,
+                              ImageFrame::kDefaultAlignmentBoundary);
+  return ::mediapipe::OkStatus();
+}
+
+}  // namespace autoflip
+}  // namespace mediapipe

mediapipe/examples/desktop/autoflip/quality/padding_effect_generator.h

@@ -0,0 +1,70 @@
+#ifndef MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_PADDING_EFFECT_GENERATOR_H_
+#define MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_PADDING_EFFECT_GENERATOR_H_
+
+#include "mediapipe/framework/formats/image_frame.h"
+#include "mediapipe/framework/port/opencv_core_inc.h"
+#include "mediapipe/framework/port/opencv_imgproc_inc.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+namespace autoflip {
+
+// Generates padding effects given input frames. Depending on where the padded
+// contents are added, there are two cases:
+// 1) Pad on the top and bottom of the input frame, aka vertical padding, i.e.
+//    input_aspect_ratio > target_aspect_ratio. In this case, output frames will
+//    have the same height as input frames, and the width will be adjusted to
+//    match the target aspect ratio.
+// 2) Pad on the left and right of the input frame, aka horizontal padding, i.e.
+//    input_aspect_ratio < target_aspect_ratio. In this case, output frames will
+//    have the same width as original frames, and the height will be adjusted to
+//    match the target aspect ratio.
+// If a background color is given, the background of the output frame will be
+// filled with this solid color; otherwise, it is a blurred version of the input
+// frame.
+//
+// Note: in both horizontal and vertical padding effects, the output frame size
+// will be at most as large as the input frame size, with one dimension the
+// same as the input (horizontal padding: width, vertical padding: height). If
+// you intented to have the output frame be larger, you could add a
+// ScaleImageCalculator as an upstream node before calling this calculator in
+// your MediaPipe graph (not as a downstream node, because visual details may
+// lose after appling the padding effect).
+class PaddingEffectGenerator {
+ public:
+  // Always outputs width and height that are divisible by 2 if
+  // scale_to_multiple_of_two is set to true.
+  PaddingEffectGenerator(const int input_width, const int input_height,
+                         const double target_aspect_ratio,
+                         bool scale_to_multiple_of_two = false);
+
+  // Apply the padding effect on the input frame.
+  // - blur_cv_size: The cv::Size() parameter used in creating blurry effects
+  //   for padding backgrounds.
+  // - background_contrast: Contrast adjustment for padding background. This
+  //   value should between 0 and 1, and the smaller the value, the darker the
+  //   background.
+  // - overlay_opacity: In addition to adjusting the contrast, a translucent
+  //   black layer will be alpha blended with the background. This value defines
+  //   the opacity of the black layer.
+  // - background_color_in_rgb: If not null, uses this solid color as background
+  //   instead of blurring the image, and does not adjust contrast or opacity.
+  ::mediapipe::Status Process(
+      const ImageFrame& input_frame, const float background_contrast,
+      const int blur_cv_size, const float overlay_opacity,
+      ImageFrame* output_frame,
+      const cv::Scalar* background_color_in_rgb = nullptr);
+
+ private:
+  double target_aspect_ratio_;
+  int input_width_ = -1;
+  int input_height_ = -1;
+  int output_width_ = -1;
+  int output_height_ = -1;
+  bool is_vertical_padding_;
+};
+
+}  // namespace autoflip
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_PADDING_EFFECT_GENERATOR_H_

mediapipe/examples/desktop/autoflip/quality/padding_effect_generator_test.cc

@@ -0,0 +1,187 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/examples/desktop/autoflip/quality/padding_effect_generator.h"
+
+#include "absl/strings/str_cat.h"
+#include "mediapipe/framework/deps/file_path.h"
+#include "mediapipe/framework/formats/image_frame.h"
+#include "mediapipe/framework/formats/image_frame_opencv.h"
+#include "mediapipe/framework/port/commandlineflags.h"
+#include "mediapipe/framework/port/file_helpers.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/opencv_imgcodecs_inc.h"
+#include "mediapipe/framework/port/opencv_imgproc_inc.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status_builder.h"
+#include "mediapipe/framework/port/status_matchers.h"
+
+DEFINE_string(input_image, "", "The path to an input image.");
+DEFINE_string(output_folder, "", "The folder to output test result images.");
+
+namespace mediapipe {
+namespace autoflip {
+namespace {
+
+// An 320x180 RGB test image.
+constexpr char kTestImage[] =
+    "mediapipe/examples/desktop/autoflip/quality/testdata/"
+    "google.jpg";
+constexpr char kResultImagePrefix[] =
+    "mediapipe/examples/desktop/autoflip/quality/testdata/"
+    "result_";
+
+const cv::Scalar kRed = cv::Scalar(255, 0, 0);
+
+void TestWithAspectRatio(const double aspect_ratio,
+                         const cv::Scalar* background_color_in_rgb = nullptr) {
+  std::string test_image;
+  const bool process_arbitrary_image = !FLAGS_input_image.empty();
+  if (!process_arbitrary_image) {
+    std::string test_image_path = mediapipe::file::JoinPath("./", kTestImage);
+    MP_ASSERT_OK(mediapipe::file::GetContents(test_image_path, &test_image));
+  } else {
+    MP_ASSERT_OK(mediapipe::file::GetContents(FLAGS_input_image, &test_image));
+  }
+
+  const std::vector<char> contents_vector(test_image.begin(), test_image.end());
+  cv::Mat decoded_mat =
+      cv::imdecode(contents_vector, -1 /* return the loaded image as-is */);
+
+  ImageFormat::Format image_format = ImageFormat::UNKNOWN;
+  cv::Mat output_mat;
+  switch (decoded_mat.channels()) {
+    case 1:
+      image_format = ImageFormat::GRAY8;
+      output_mat = decoded_mat;
+      break;
+    case 3:
+      image_format = ImageFormat::SRGB;
+      cv::cvtColor(decoded_mat, output_mat, cv::COLOR_BGR2RGB);
+      break;
+    case 4:
+      MP_ASSERT_OK(::mediapipe::UnimplementedErrorBuilder(MEDIAPIPE_LOC)
+                   << "4-channel image isn't supported yet");
+      break;
+    default:
+      MP_ASSERT_OK(::mediapipe::FailedPreconditionErrorBuilder(MEDIAPIPE_LOC)
+                   << "Unsupported number of channels: "
+                   << decoded_mat.channels());
+  }
+  std::unique_ptr<ImageFrame> test_frame = absl::make_unique<ImageFrame>(
+      image_format, decoded_mat.size().width, decoded_mat.size().height);
+  output_mat.copyTo(formats::MatView(test_frame.get()));
+
+  PaddingEffectGenerator generator(test_frame->Width(), test_frame->Height(),
+                                   aspect_ratio);
+  ImageFrame result_frame;
+  MP_ASSERT_OK(generator.Process(*test_frame, 0.3, 40, 0.0, &result_frame,
+                                 background_color_in_rgb));
+  cv::Mat original_mat = formats::MatView(&result_frame);
+  cv::Mat input_mat;
+  switch (original_mat.channels()) {
+    case 1:
+      input_mat = original_mat;
+      break;
+    case 3:
+      // OpenCV assumes the image to be BGR order. To use imencode(), do color
+      // conversion first.
+      cv::cvtColor(original_mat, input_mat, cv::COLOR_RGB2BGR);
+      break;
+    case 4:
+      MP_ASSERT_OK(::mediapipe::UnimplementedErrorBuilder(MEDIAPIPE_LOC)
+                   << "4-channel image isn't supported yet");
+      break;
+    default:
+      MP_ASSERT_OK(::mediapipe::FailedPreconditionErrorBuilder(MEDIAPIPE_LOC)
+                   << "Unsupported number of channels: "
+                   << original_mat.channels());
+  }
+
+  std::vector<int> parameters;
+  parameters.push_back(cv::IMWRITE_JPEG_QUALITY);
+  constexpr int kEncodingQuality = 75;
+  parameters.push_back(kEncodingQuality);
+
+  std::vector<uchar> encode_buffer;
+  // Note that imencode() will store the data in RGB order.
+  // Check its JpegEncoder::write() in "imgcodecs/src/grfmt_jpeg.cpp" for more
+  // info.
+  if (!cv::imencode(".jpg", input_mat, encode_buffer, parameters)) {
+    MP_ASSERT_OK(::mediapipe::InternalErrorBuilder(MEDIAPIPE_LOC)
+                 << "Fail to encode the image to be jpeg format.");
+  }
+
+  std::string output_string(absl::string_view(
+      reinterpret_cast<const char*>(&encode_buffer[0]), encode_buffer.size()));
+
+  if (!process_arbitrary_image) {
+    std::string result_string_path = mediapipe::file::JoinPath(
+        "./", absl::StrCat(kResultImagePrefix, aspect_ratio,
+                           background_color_in_rgb ? "_solid_background" : "",
+                           ".jpg"));
+    std::string result_image;
+    MP_ASSERT_OK(
+        mediapipe::file::GetContents(result_string_path, &result_image));
+    EXPECT_EQ(result_image, output_string);
+  } else {
+    std::string output_string_path = mediapipe::file::JoinPath(
+        FLAGS_output_folder,
+        absl::StrCat("result_", aspect_ratio,
+                     background_color_in_rgb ? "_solid_background" : "",
+                     ".jpg"));
+    MP_ASSERT_OK(
+        mediapipe::file::SetContents(output_string_path, output_string));
+  }
+}
+
+TEST(PaddingEffectGeneratorTest, Success) {
+  TestWithAspectRatio(0.3);
+  TestWithAspectRatio(0.6);
+  TestWithAspectRatio(1.0);
+  TestWithAspectRatio(1.6);
+  TestWithAspectRatio(2.5);
+  TestWithAspectRatio(3.4);
+}
+
+TEST(PaddingEffectGeneratorTest, SuccessWithBackgroundColor) {
+  TestWithAspectRatio(0.3, &kRed);
+  TestWithAspectRatio(0.6, &kRed);
+  TestWithAspectRatio(1.0, &kRed);
+  TestWithAspectRatio(1.6, &kRed);
+  TestWithAspectRatio(2.5, &kRed);
+  TestWithAspectRatio(3.4, &kRed);
+}
+
+TEST(PaddingEffectGeneratorTest, ScaleToMultipleOfTwo) {
+  int input_width = 30;
+  int input_height = 30;
+  double target_aspect_ratio = 0.5;
+  int expect_width = 14;
+  int expect_height = input_height;
+  auto test_frame = absl::make_unique<ImageFrame>(/*format=*/ImageFormat::SRGB,
+                                                  input_width, input_height);
+
+  PaddingEffectGenerator generator(test_frame->Width(), test_frame->Height(),
+                                   target_aspect_ratio,
+                                   /*scale_to_multiple_of_two=*/true);
+  ImageFrame result_frame;
+  MP_ASSERT_OK(generator.Process(*test_frame, 0.3, 40, 0.0, &result_frame));
+  EXPECT_EQ(result_frame.Width(), expect_width);
+  EXPECT_EQ(result_frame.Height(), expect_height);
+}
+}  // namespace
+}  // namespace autoflip
+}  // namespace mediapipe

mediapipe/examples/desktop/autoflip/quality/piecewise_linear_function.cc

@@ -0,0 +1,69 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/examples/desktop/autoflip/quality/piecewise_linear_function.h"
+
+#include <stddef.h>
+
+#include <algorithm>
+#include <limits>
+#include <vector>
+
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+namespace autoflip {
+
+void PiecewiseLinearFunction::AddPoint(double x, double y) {
+  if (!points_.empty()) {
+    CHECK_GE(x, points_.back().x)
+        << "Points must be provided in non-decreasing x order.";
+  }
+  points_.push_back(PiecewiseLinearFunction::Point(x, y));
+}
+
+std::vector<PiecewiseLinearFunction::Point>::const_iterator
+PiecewiseLinearFunction::GetIntervalIterator(double input) const {
+  PiecewiseLinearFunction::Point input_point(input, 0);
+  std::vector<PiecewiseLinearFunction::Point>::const_iterator iter =
+      std::lower_bound(points_.begin(), points_.end(), input_point,
+                       PointCompare());
+  return iter;
+}
+
+double PiecewiseLinearFunction::Interpolate(
+    const PiecewiseLinearFunction::Point& p1,
+    const PiecewiseLinearFunction::Point& p2, double input) const {
+  CHECK_LT(p1.x, input);
+  CHECK_GE(p2.x, input);
+
+  return p2.y - (p2.x - input) / (p2.x - p1.x) * (p2.y - p1.y);
+}
+
+double PiecewiseLinearFunction::Evaluate(double const input) const {
+  std::vector<PiecewiseLinearFunction::Point>::const_iterator i =
+      GetIntervalIterator(input);
+  if (i == points_.begin()) {
+    return points_.front().y;
+  }
+  if (i == points_.end()) {
+    return points_.back().y;
+  }
+
+  std::vector<PiecewiseLinearFunction::Point>::const_iterator prev = i - 1;
+  return Interpolate(*prev, *i, input);
+}
+
+}  // namespace autoflip
+}  // namespace mediapipe

mediapipe/examples/desktop/autoflip/quality/piecewise_linear_function.h

@@ -0,0 +1,82 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_PIECEWISE_LINEAR_FUNCTION_H_
+#define MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_PIECEWISE_LINEAR_FUNCTION_H_
+
+#include <vector>
+
+namespace mediapipe {
+namespace autoflip {
+
+// Implementation of piecewise linear functions. The function is specified as a
+// series of points (x1,y1), (x2,y2),..., (xn,yn). It can be constructed
+// programmatically by repeatedly calling the AddPoint(x, y) method.
+class PiecewiseLinearFunction {
+ public:
+  PiecewiseLinearFunction() {}
+
+  // Evaluate the function at the specified input. The output
+  // saturates at the values of the first and last interpolation
+  // points.
+  // f(x) = y1 for x <= x1   // Saturate at the lowest value
+  // f(x) = yn for x >  xn   // Saturate at the highest value
+  // f(x) = (x-xj)/(xk-xj)*(yk-yj) + yk for xj < x <= xk and k = j+1
+  double Evaluate(double input) const;
+
+  // Adds the given point to the function.  Points must be added in
+  // non-decreasing x order.  Because the points are given in sorted
+  // order, this function can be used to construct discontinuous
+  // functions.  For example, if one defines
+  //    f.AddPoint(-1.0, 0.0)
+  //    f.AddPoint( 0.0, 0.0)
+  //    f.AddPoint( 0.0, 1.0)
+  //    f.AddPoint( 1.0, 1.0)
+  // the result function f is discontinuous at 0.0.  By convention,
+  // this function will return f.Evaluate(0.0) = 0.0, and
+  // f.Evaluate(1e-12) = 1.0.  This convention corresponds to the
+  // natural behavior of GetIntervalIterator().
+  void AddPoint(double x, double y);
+
+ private:
+  struct Point {
+    double x;
+    double y;
+    Point(double X, double Y) : x(X), y(Y) {}
+  };
+
+  // A functor for use with stl algorithms like sort() and lower_bound() that
+  // sorts by the point's x value.
+  class PointCompare {
+   public:
+    bool operator()(const Point& p1, const Point& p2) const {
+      return p1.x < p2.x;
+    }
+  };
+
+  // Returns the iterator, i, closest to points_.begin() such that
+  // input <= i->x or it returns points_.end() if input > all x values
+  // in points_.
+  std::vector<Point>::const_iterator GetIntervalIterator(double input) const;
+
+  // Given two points p1 and p2 such that p1.x < input and p2.x >= input this
+  // returns the linear interpolation of the y value.
+  double Interpolate(const Point& p1, const Point& p2, double input) const;
+
+  std::vector<Point> points_;
+};
+
+}  // namespace autoflip
+}  // namespace mediapipe
+#endif  // MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_PIECEWISE_LINEAR_FUNCTION_H_

mediapipe/examples/desktop/autoflip/quality/piecewise_linear_function_test.cc

@@ -0,0 +1,73 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/examples/desktop/autoflip/quality/piecewise_linear_function.h"
+
+#include <stddef.h>
+
+#include <memory>
+
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace {
+
+using mediapipe::autoflip::PiecewiseLinearFunction;
+
+// It should be OK to pass a spec that's out of order as it gets sorted.
+TEST(PiecewiseLinearFunctionTest, ReordersSpec) {
+  PiecewiseLinearFunction f;
+  // This defines the line y = x between 0 and 5
+  f.AddPoint(0, 0);
+  f.AddPoint(1, 1);
+  f.AddPoint(2, 2);
+  f.AddPoint(3, 3);
+  f.AddPoint(5, 5);
+
+  // Should be 0 as -1 is less than the smallest x value in the spec so it
+  // should saturate.
+  ASSERT_EQ(0, f.Evaluate(-1));
+
+  // These shoud all be on the line y = x
+  ASSERT_EQ(0, f.Evaluate(0));
+  ASSERT_EQ(0.5, f.Evaluate(0.5));
+  ASSERT_EQ(4.5, f.Evaluate(4.5));
+  ASSERT_EQ(5, f.Evaluate(5));
+
+  // Saturating on the high end.
+  ASSERT_EQ(5, f.Evaluate(6));
+}
+
+TEST(PiecewiseLinearFunctionTest, TestAddPoints) {
+  PiecewiseLinearFunction function;
+  function.AddPoint(0.0, 0.0);
+  function.AddPoint(1.0, 1.0);
+  EXPECT_DOUBLE_EQ(0.0, function.Evaluate(-1.0));
+  EXPECT_DOUBLE_EQ(0.0, function.Evaluate(0.0));
+  EXPECT_DOUBLE_EQ(0.25, function.Evaluate(0.25));
+}
+
+TEST(PiecewiseLinearFunctionTest, AddPointsDiscontinuous) {
+  PiecewiseLinearFunction function;
+  function.AddPoint(-1.0, 0.0);
+  function.AddPoint(0.0, 0.0);
+  function.AddPoint(0.0, 1.0);
+  function.AddPoint(1.0, 1.0);
+  EXPECT_DOUBLE_EQ(0.0, function.Evaluate(-1.0));
+  EXPECT_DOUBLE_EQ(0.0, function.Evaluate(0.0));
+  EXPECT_DOUBLE_EQ(1.0, function.Evaluate(1e-12));
+  EXPECT_DOUBLE_EQ(1.0, function.Evaluate(3.14));
+}
+
+}  // namespace

mediapipe/examples/desktop/autoflip/quality/polynomial_regression_path_solver.cc

@@ -0,0 +1,160 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/examples/desktop/autoflip/quality/polynomial_regression_path_solver.h"
+
+#include "ceres/autodiff_cost_function.h"
+#include "ceres/cost_function.h"
+#include "ceres/loss_function.h"
+#include "ceres/solver.h"
+#include "mediapipe/examples/desktop/autoflip/quality/focus_point.pb.h"
+#include "mediapipe/framework/port/opencv_core_inc.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+namespace autoflip {
+
+using ceres::AutoDiffCostFunction;
+using ceres::CauchyLoss;
+using ceres::CostFunction;
+using ceres::Problem;
+using ceres::Solve;
+using ceres::Solver;
+
+namespace {
+
+// A residual operator that computes the error using polynomial fitting.
+struct PolynomialResidual {
+  PolynomialResidual(double in, double out) : in_(in), out_(out) {}
+
+  template <typename T>
+  bool operator()(const T* const a, const T* const b, const T* const c,
+                  const T* const d, const T* const k, T* residual) const {
+    residual[0] = out_ - a[0] * in_ - b[0] * in_ * in_ -
+                  c[0] * in_ * in_ * in_ - d[0] * in_ * in_ * in_ * in_ - k[0];
+    return true;
+  }
+
+ private:
+  const double in_;
+  const double out_;
+};
+
+float ComputeDelta(const float in, const int original_dimension,
+                   const int output_dimension, const double a, const double b,
+                   const double c, const double d, const double k) {
+  float out =
+      a * in + b * in * in + c * in * in * in + d * in * in * in * in + k;
+  float delta = (out - 0.5) * 2 * output_dimension;
+  const float max_delta = (original_dimension - output_dimension) / 2.0f;
+
+  // Make sure delta doesn't move the camera off the frame boundary.
+  if (delta > max_delta) {
+    delta = max_delta;
+  } else if (delta < -max_delta) {
+    delta = -max_delta;
+  }
+  return delta;
+}
+}  // namespace
+
+void PolynomialRegressionPathSolver::AddCostFunctionToProblem(
+    const double in, const double out, Problem* problem, double* a, double* b,
+    double* c, double* d, double* k) {
+  // Creating a cost function, with 1D residual and 5 1D parameter blocks. This
+  // is what the "1, 1, 1, 1, 1, 1" std::string below means.
+  CostFunction* cost_function =
+      new AutoDiffCostFunction<PolynomialResidual, 1, 1, 1, 1, 1, 1>(
+          new PolynomialResidual(in, out));
+  VLOG(1) << "------- adding " << in << ": " << out;
+  problem->AddResidualBlock(cost_function, new CauchyLoss(0.5), a, b, c, d, k);
+}
+
+::mediapipe::Status PolynomialRegressionPathSolver::ComputeCameraPath(
+    const std::vector<FocusPointFrame>& focus_point_frames,
+    const std::vector<FocusPointFrame>& prior_focus_point_frames,
+    const int original_width, const int original_height, const int output_width,
+    const int output_height, std::vector<cv::Mat>* all_xforms) {
+  RET_CHECK_GE(original_width, output_width);
+  RET_CHECK_GE(original_height, output_height);
+  const bool should_solve_x_problem = original_width != output_width;
+  const bool should_solve_y_problem = original_height != output_height;
+  RET_CHECK_GT(focus_point_frames.size() + prior_focus_point_frames.size(), 0);
+  Problem problem_x, problem_y;
+  for (int i = 0; i < prior_focus_point_frames.size(); ++i) {
+    const auto& spf = prior_focus_point_frames[i];
+    for (const auto& sp : spf.point()) {
+      const double center_x = sp.norm_point_x();
+      const double center_y = sp.norm_point_y();
+      const auto t = i;
+      if (should_solve_x_problem) {
+        AddCostFunctionToProblem(t, center_x, &problem_x, &xa_, &xb_, &xc_,
+                                 &xd_, &xk_);
+      }
+      if (should_solve_y_problem) {
+        AddCostFunctionToProblem(t, center_y, &problem_y, &ya_, &yb_, &yc_,
+                                 &yd_, &yk_);
+      }
+    }
+  }
+  for (int i = 0; i < focus_point_frames.size(); ++i) {
+    const auto& spf = focus_point_frames[i];
+    for (const auto& sp : spf.point()) {
+      const double center_x = sp.norm_point_x();
+      const double center_y = sp.norm_point_y();
+      const auto t = i + prior_focus_point_frames.size();
+      if (should_solve_x_problem) {
+        AddCostFunctionToProblem(t, center_x, &problem_x, &xa_, &xb_, &xc_,
+                                 &xd_, &xk_);
+      }
+      if (should_solve_y_problem) {
+        AddCostFunctionToProblem(t, center_y, &problem_y, &ya_, &yb_, &yc_,
+                                 &yd_, &yk_);
+      }
+    }
+  }
+
+  Solver::Options options;
+  options.linear_solver_type = ceres::DENSE_QR;
+
+  Solver::Summary summary;
+  Solve(options, &problem_x, &summary);
+  all_xforms->clear();
+  for (int i = 0;
+       i < focus_point_frames.size() + prior_focus_point_frames.size(); i++) {
+    // Code below assigns values into an affine model, defined as:
+    //  [1 0 dx]
+    //  [0 1 dy]
+    // When the camera moves along x axis, we assign delta to dx; otherwise we
+    // assign delta to dy.
+    cv::Mat transform = cv::Mat::eye(2, 3, CV_32FC1);
+    const float in = static_cast<float>(i);
+    if (should_solve_x_problem) {
+      const float delta = ComputeDelta(in, original_width, output_width, xa_,
+                                       xb_, xc_, xd_, xk_);
+      transform.at<float>(0, 2) = delta;
+    }
+    if (should_solve_y_problem) {
+      const float delta = ComputeDelta(in, original_height, output_height, ya_,
+                                       yb_, yc_, yd_, yk_);
+      transform.at<float>(1, 2) = delta;
+    }
+    all_xforms->push_back(transform);
+  }
+  return mediapipe::OkStatus();
+}
+
+}  // namespace autoflip
+}  // namespace mediapipe

mediapipe/examples/desktop/autoflip/quality/polynomial_regression_path_solver.h

@@ -0,0 +1,69 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_POLYNOMIAL_REGRESSION_PATH_SOLVER_H_
+#define MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_POLYNOMIAL_REGRESSION_PATH_SOLVER_H_
+
+#include "ceres/problem.h"
+#include "mediapipe/examples/desktop/autoflip/quality/focus_point.pb.h"
+#include "mediapipe/framework/port/opencv_core_inc.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+namespace autoflip {
+
+class PolynomialRegressionPathSolver {
+ public:
+  PolynomialRegressionPathSolver()
+      : xa_(0.0),
+        xb_(0.0),
+        xc_(0.0),
+        xd_(0.0),
+        xk_(0.0),
+        ya_(0.0),
+        yb_(0.0),
+        yc_(0.0),
+        yd_(0.0),
+        yk_(0.0) {}
+
+  // Given a series of focus points on frames, uses polynomial regression to
+  // compute a best guess of a 1D camera movement trajectory along x-axis and
+  // y-axis, such that focus points can be preserved as much as possible. The
+  // returned |all_xforms| hold the camera location at each timestamp
+  // corresponding to each input frame.
+  ::mediapipe::Status ComputeCameraPath(
+      const std::vector<FocusPointFrame>& focus_point_frames,
+      const std::vector<FocusPointFrame>& prior_focus_point_frames,
+      const int original_width, const int original_height,
+      const int output_width, const int output_height,
+      std::vector<cv::Mat>* all_xforms);
+
+ private:
+  // Adds a new cost function, constructed using |in| and |out|, into |problem|.
+  void AddCostFunctionToProblem(const double in, const double out,
+                                ceres::Problem* problem, double* a, double* b,
+                                double* c, double* d, double* k);
+
+  // The current implementation fixes the polynomial order at 4, i.e. the
+  // equation to estimate is: out = a * in + b * in^2 + c * in^3 + d * in^4 + k.
+  // The two sets of parameters below are for estimating trajectories along
+  // x-axis and y-axis, respectively.
+  double xa_, xb_, xc_, xd_, xk_;
+  double ya_, yb_, yc_, yd_, yk_;
+};
+
+}  // namespace autoflip
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_POLYNOMIAL_REGRESSION_PATH_SOLVER_H_

mediapipe/examples/desktop/autoflip/quality/polynomial_regression_path_solver_test.cc

@@ -0,0 +1,246 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/examples/desktop/autoflip/quality/polynomial_regression_path_solver.h"
+
+#include "mediapipe/examples/desktop/autoflip/quality/focus_point.pb.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/opencv_core_inc.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/port/status_matchers.h"
+
+namespace mediapipe {
+namespace autoflip {
+namespace {
+
+// A series of focus point locations from a real video.
+constexpr int kNumObservations = 291;
+constexpr double data[] = {
+    1,   0.4072740648, 2,   0.406096287,  3,   0.4049185093, 4,   0.4037407361,
+    5,   0.402562963,  6,   0.4013851806, 7,   0.4002074074, 8,   0.399029625,
+    9,   0.3978518519, 10,  0.3966740741, 11,  0.3954963056, 12,  0.3943185231,
+    13,  0.39314075,   14,  0.3919629676, 15,  0.3907851944, 16,  0.3896074167,
+    17,  0.3884296435, 18,  0.3872518611, 19,  0.386074088,  20,  0.3848963194,
+    21,  0.3837185417, 22,  0.3825407685, 23,  0.3813629861, 24,  0.3801852037,
+    25,  0.3790074398, 26,  0.3778296574, 27,  0.376651875,  28,  0.3754741111,
+    29,  0.3742963287, 30,  0.3731185509, 31,  0.3719407685, 32,  0.3707629861,
+    33,  0.3695852222, 34,  0.3684074398, 35,  0.3672296759, 36,  0.3661564352,
+    37,  0.3651877315, 38,  0.3643235509, 39,  0.3635639213, 40,  0.3629088241,
+    41,  0.36235825,   42,  0.3619122222, 43,  0.3615707269, 44,  0.3613337639,
+    45,  0.3612013519, 46,  0.3611734583, 47,  0.3612500926, 48,  0.3614312778,
+    49,  0.361717,     50,  0.3621072546, 51,  0.362602037,  52,  0.3632013519,
+    53,  0.3639052083, 54,  0.3647136111, 55,  0.3656265417, 56,  0.3666440046,
+    57,  0.3677659907, 58,  0.3689925139, 59,  0.3703235926, 60,  0.3716546528,
+    61,  0.3729857269, 62,  0.374316787,  63,  0.3756478611, 64,  0.3769789259,
+    65,  0.37831,      66,  0.3796410648, 67,  0.3809721296, 68,  0.3823031944,
+    69,  0.3836342685, 70,  0.384965338,  71,  0.3862963981, 72,  0.3876274676,
+    73,  0.388958537,  74,  0.3902290324, 75,  0.391438963,  76,  0.3925883241,
+    77,  0.3936771204, 78,  0.3947053426, 79,  0.3956729954, 80,  0.396580088,
+    81,  0.3974265972, 82,  0.3982125509, 83,  0.3989379259, 84,  0.3996027407,
+    85,  0.4002069815, 86,  0.400750662,  87,  0.4012337639, 88,  0.4016562963,
+    89,  0.4020182731, 90,  0.4023196667, 91,  0.4025604954, 92,  0.4027407593,
+    93,  0.4028604491, 94,  0.4029195787, 95,  0.4029181296, 96,  0.4028561204,
+    97,  0.4027407593, 98,  0.4026254028, 99,  0.4025100417, 100, 0.4023946852,
+    101, 0.4022793241, 102, 0.402163963,  103, 0.4020486065, 104, 0.4019332454,
+    105, 0.4018178889, 106, 0.4017025278, 107, 0.4015871759, 108, 0.4014718102,
+    109, 0.4013564491, 110, 0.4012410972, 111, 0.4011257315, 112, 0.4010103704,
+    113, 0.4008950185, 114, 0.400779662,  115, 0.4008743935, 116, 0.4011792083,
+    117, 0.4016941157, 118, 0.4024191111, 119, 0.4033541944, 120, 0.4044993704,
+    121, 0.4058546343, 122, 0.4074199815, 123, 0.4091954259, 124, 0.4111809537,
+    125, 0.4133765741, 126, 0.415782287,  127, 0.4183980787, 128, 0.4212239676,
+    129, 0.4242599352, 130, 0.427506,     131, 0.4309621528, 132, 0.4346283935,
+    133, 0.4385047222, 134, 0.4425911407, 135, 0.4468876481, 136, 0.4513942454,
+    137, 0.4561109306, 138, 0.4608276204, 139, 0.4655443056, 140, 0.4702609861,
+    141, 0.4749776736, 142, 0.4796943574, 143, 0.4844110435, 144, 0.4891277287,
+    145, 0.4938444148, 146, 0.4985611,    147, 0.5032777852, 148, 0.5079944708,
+    149, 0.512711156,  150, 0.5174278403, 151, 0.5221445259, 152, 0.526861212,
+    153, 0.5315778981, 154, 0.5362945833, 155, 0.5410112662, 156, 0.5457279537,
+    157, 0.5504446435, 158, 0.5551613241, 159, 0.5598780093, 160, 0.5643503102,
+    161, 0.5685782454, 162, 0.572561787,  163, 0.5763009583, 164, 0.5797957546,
+    165, 0.583046162,  166, 0.5860521944, 167, 0.5888138611, 168, 0.5913311296,
+    169, 0.5936040278, 170, 0.5956325463, 171, 0.5974166806, 172, 0.5989564491,
+    173, 0.6002518287, 174, 0.6013028241, 175, 0.6021094491, 176, 0.6026716944,
+    177, 0.6029895556, 178, 0.6030630556, 179, 0.602892162,  180, 0.6024768889,
+    181, 0.6018172361, 182, 0.6009132083, 183, 0.5997648009, 184, 0.5983720139,
+    185, 0.5967348519, 186, 0.595057662,  187, 0.5933804769, 188, 0.5917032963,
+    189, 0.5900261065, 190, 0.588348912,  191, 0.5866717315, 192, 0.5849945417,
+    193, 0.5833173519, 194, 0.5816401713, 195, 0.5799629861, 196, 0.578285787,
+    197, 0.5766086111, 198, 0.5749314213, 199, 0.5732542315, 200, 0.5715770509,
+    201, 0.5698998611, 202, 0.5682226667, 203, 0.5665454861, 204, 0.5648682963,
+    205, 0.5631911157, 206, 0.5615139213, 207, 0.559989287,  208, 0.5586172083,
+    209, 0.5573976713, 210, 0.5563306944, 211, 0.5554162662, 212, 0.5546543889,
+    213, 0.5540450648, 214, 0.5535882917, 215, 0.5532840694, 216, 0.5531324028,
+    217, 0.5531332824, 218, 0.5532867176, 219, 0.5535927083, 220, 0.5540512454,
+    221, 0.5546623333, 222, 0.5554259769, 223, 0.5563421667, 224, 0.5574109148,
+    225, 0.5586322083, 226, 0.5600060556, 227, 0.5615324583, 228, 0.563211412,
+    229, 0.565042912,  230, 0.5670269722, 231, 0.5691635833, 232, 0.5714527315,
+    233, 0.5738944491, 234, 0.576488713,  235, 0.5792355231, 236, 0.581982338,
+    237, 0.5847291528, 238, 0.5874759676, 239, 0.5902227824, 240, 0.5929695972,
+    241, 0.5957164074, 242, 0.5984632269, 243, 0.601210037,  244, 0.6039568565,
+    245, 0.6067036667, 246, 0.6094504815, 247, 0.6121973056, 248, 0.6149441111,
+    249, 0.6176909352, 250, 0.62043775,   251, 0.6231845556, 252, 0.6258408148,
+    253, 0.628406537,  254, 0.6308817269, 255, 0.6332663426, 256, 0.6355604167,
+    257, 0.6377639352, 258, 0.6398769259, 259, 0.6418993519, 260, 0.6438312407,
+    261, 0.6456725787, 262, 0.6474233704, 263, 0.6490836111, 264, 0.650653287,
+    265, 0.6521324444, 266, 0.653521037,  267, 0.6548190926, 268, 0.656026588,
+    269, 0.6571435417, 270, 0.6581699537, 271, 0.6591058102, 272, 0.6599511204,
+    273, 0.6607058796, 274, 0.6613700926, 275, 0.6620343056, 276, 0.6626985185,
+    277, 0.6633627454, 278, 0.6640269537, 279, 0.6646911759, 280, 0.6653553843,
+    281, 0.6660195972, 282, 0.6666838056, 283, 0.6673480278, 284, 0.6680122361,
+    285, 0.668676463,  286, 0.6693406713, 287, 0.6700048935, 288, 0.6706691019,
+    289, 0.6713333333, 290, 0.671997537,  291, 0.6726617454,
+};
+
+constexpr double prediction[] = {
+    18.885935,   16.560495,  14.316487,  12.15229,   10.066307,   8.056951,
+    6.1226487,   4.2618394,  2.4729967,  0.7545829,  -0.894928,   -2.47702,
+    -3.9931893,  -5.4449024, -6.833619,  -8.160782,  -9.427822,   -10.63615,
+    -11.78717,   -12.882269, -13.922811, -14.910168, -15.845669,  -16.730648,
+    -17.566418,  -18.354284, -19.095533, -19.791435, -20.443249,  -21.052212,
+    -21.619564,  -22.146511, -22.634256, -23.08399,  -23.496883,  -23.874086,
+    -24.216753,  -24.526012, -24.80297,  -25.048738, -25.264395,  -25.451015,
+    -25.609661,  -25.74137,  -25.84718,  -25.928093, -25.985123,  -26.01925,
+    -26.031458,  -26.022684, -25.993896, -25.946003, -25.879932,  -25.796581,
+    -25.696838,  -25.581581, -25.451654, -25.307919, -25.151188,  -24.982292,
+    -24.802023,  -24.611176, -24.410517, -24.200804, -23.98278,   -23.757189,
+    -23.52473,   -23.286121, -23.042034, -22.79315,  -22.540123,  -22.283602,
+    -22.024214,  -21.762579, -21.499294, -21.234953, -20.970118,  -20.70536,
+    -20.441223,  -20.178228, -19.916899, -19.65773,  -19.401217,  -19.147831,
+    -18.898027,  -18.65226,  -18.410952, -18.174517, -17.943365,  -17.717875,
+    -17.498428,  -17.285383, -17.079079, -16.879856, -16.688019,  -16.503883,
+    -16.327726,  -16.15982,  -16.000439, -15.849811, -15.7081785, -15.575754,
+    -15.452737,  -15.339321, -15.235674, -15.141964, -15.058327,  -14.9848995,
+    -14.9218025, -14.869129, -14.826971, -14.795404, -14.774489,  -14.764267,
+    -14.764768,  -14.776015, -14.798016, -14.830744, -14.874178,  -14.9282875,
+    -14.993011,  -15.068281, -15.15401,  -15.250105, -15.356457,  -15.472937,
+    -15.599405,  -15.73571,  -15.881681, -16.03713,  -16.201872,  -16.37569,
+    -16.558355,  -16.749626, -16.94926,  -17.156982, -17.372507,  -17.595535,
+    -17.825771,  -18.062872, -18.306505, -18.55632,  -18.811947,  -19.072998,
+    -19.339085,  -19.609785, -19.884687, -20.16334,  -20.4453,    -20.730091,
+    -21.017235,  -21.306234, -21.59658,  -21.887743, -22.179192,  -22.470362,
+    -22.760695,  -23.049604, -23.336494, -23.620754, -23.901754,  -24.17887,
+    -24.451435,  -24.718779, -24.980236, -25.235092, -25.482649,  -25.722181,
+    -25.952942,  -26.174181, -26.38514,  -26.585024, -26.77304,   -26.948387,
+    -27.110231,  -27.257734, -27.39005,  -27.506304, -27.605618,  -27.68709,
+    -27.749819,  -27.792877, -27.81533,  -27.816212, -27.794569,  -27.749413,
+    -27.679752,  -27.584576, -27.462858, -27.313555, -27.135622,  -26.927996,
+    -26.689583,  -26.419294, -26.11602,  -25.778639, -25.40601,   -24.996979,
+    -24.550379,  -24.06503,
+};
+
+void GenerateDataPoints(
+    const int focus_point_frames_length,
+    const int prior_focus_point_frames_length,
+    std::vector<FocusPointFrame>* focus_point_frames,
+    std::vector<FocusPointFrame>* prior_focus_point_frames) {
+  CHECK(focus_point_frames_length + prior_focus_point_frames_length <=
+        kNumObservations);
+  for (int i = 0; i < prior_focus_point_frames_length; i++) {
+    FocusPoint sp;
+    sp.set_norm_point_x(data[i]);
+    FocusPointFrame spf;
+    *spf.add_point() = sp;
+    prior_focus_point_frames->push_back(spf);
+  }
+  for (int i = 0; i < focus_point_frames_length; i++) {
+    FocusPoint sp;
+    sp.set_norm_point_x(data[i]);
+    FocusPointFrame spf;
+    *spf.add_point() = sp;
+    focus_point_frames->push_back(spf);
+  }
+}
+
+TEST(PolynomialRegressionPathSolverTest, Success) {
+  PolynomialRegressionPathSolver solver;
+  std::vector<FocusPointFrame> focus_point_frames;
+  std::vector<FocusPointFrame> prior_focus_point_frames;
+  std::vector<cv::Mat> all_xforms;
+  GenerateDataPoints(/* focus_point_frames_length = */ 100,
+                     /* prior_focus_point_frames_length = */ 100,
+                     &focus_point_frames, &prior_focus_point_frames);
+  constexpr int kFrameWidth = 200;
+  constexpr int kFrameHeight = 300;
+  constexpr int kCropWidth = 100;
+  constexpr int kCropHeight = 300;
+  MP_ASSERT_OK(solver.ComputeCameraPath(
+      focus_point_frames, prior_focus_point_frames, kFrameWidth, kFrameHeight,
+      kCropWidth, kCropHeight, &all_xforms));
+  ASSERT_EQ(all_xforms.size(), 200);
+  for (int i = 0; i < all_xforms.size(); i++) {
+    cv::Mat mat = all_xforms[i];
+    EXPECT_FLOAT_EQ(mat.at<float>(0, 2), prediction[i]);
+  }
+}
+
+TEST(PolynomialRegressionPathSolverTest, FewFramesShouldWork) {
+  PolynomialRegressionPathSolver solver;
+  std::vector<FocusPointFrame> focus_point_frames;
+  std::vector<FocusPointFrame> prior_focus_point_frames;
+  std::vector<cv::Mat> all_xforms;
+  GenerateDataPoints(/* focus_point_frames_length = */ 1,
+                     /* prior_focus_point_frames_length = */ 1,
+                     &focus_point_frames, &prior_focus_point_frames);
+  constexpr int kFrameWidth = 200;
+  constexpr int kFrameHeight = 300;
+  constexpr int kCropWidth = 100;
+  constexpr int kCropHeight = 300;
+  MP_ASSERT_OK(solver.ComputeCameraPath(
+      focus_point_frames, prior_focus_point_frames, kFrameWidth, kFrameHeight,
+      kCropWidth, kCropHeight, &all_xforms));
+  ASSERT_EQ(all_xforms.size(), 2);
+}
+
+TEST(PolynomialRegressionPathSolverTest, OneCurrentFrameShouldWork) {
+  PolynomialRegressionPathSolver solver;
+  std::vector<FocusPointFrame> focus_point_frames;
+  std::vector<FocusPointFrame> prior_focus_point_frames;
+  std::vector<cv::Mat> all_xforms;
+  GenerateDataPoints(/* focus_point_frames_length = */ 1,
+                     /* prior_focus_point_frames_length = */ 0,
+                     &focus_point_frames, &prior_focus_point_frames);
+  constexpr int kFrameWidth = 200;
+  constexpr int kFrameHeight = 300;
+  constexpr int kCropWidth = 100;
+  constexpr int kCropHeight = 300;
+  MP_ASSERT_OK(solver.ComputeCameraPath(
+      focus_point_frames, prior_focus_point_frames, kFrameWidth, kFrameHeight,
+      kCropWidth, kCropHeight, &all_xforms));
+  ASSERT_EQ(all_xforms.size(), 1);
+}
+
+TEST(PolynomialRegressionPathSolverTest, ZeroFrameShouldFail) {
+  PolynomialRegressionPathSolver solver;
+  std::vector<FocusPointFrame> focus_point_frames;
+  std::vector<FocusPointFrame> prior_focus_point_frames;
+  std::vector<cv::Mat> all_xforms;
+  GenerateDataPoints(/* focus_point_frames_length = */ 0,
+                     /* prior_focus_point_frames_length = */ 0,
+                     &focus_point_frames, &prior_focus_point_frames);
+  constexpr int kFrameWidth = 200;
+  constexpr int kFrameHeight = 300;
+  constexpr int kCropWidth = 100;
+  constexpr int kCropHeight = 300;
+  ASSERT_FALSE(solver
+                   .ComputeCameraPath(focus_point_frames,
+                                      prior_focus_point_frames, kFrameWidth,
+                                      kFrameHeight, kCropWidth, kCropHeight,
+                                      &all_xforms)
+                   .ok());
+}
+
+}  // namespace
+}  // namespace autoflip
+}  // namespace mediapipe

mediapipe/examples/desktop/autoflip/quality/scene_camera_motion_analyzer.cc

@@ -0,0 +1,428 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/examples/desktop/autoflip/quality/scene_camera_motion_analyzer.h"
+
+#include <limits>
+
+#include "absl/memory/memory.h"
+#include "absl/strings/str_cat.h"
+#include "absl/strings/str_format.h"
+#include "mediapipe/examples/desktop/autoflip/quality/math_utils.h"
+#include "mediapipe/examples/desktop/autoflip/quality/piecewise_linear_function.h"
+#include "mediapipe/examples/desktop/autoflip/quality/utils.h"
+#include "mediapipe/framework/port/integral_types.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/timestamp.h"
+
+namespace mediapipe {
+namespace autoflip {
+
+::mediapipe::Status
+SceneCameraMotionAnalyzer::AnalyzeSceneAndPopulateFocusPointFrames(
+    const std::vector<KeyFrameInfo>& key_frame_infos,
+    const KeyFrameCropOptions& key_frame_crop_options,
+    const std::vector<KeyFrameCropResult>& key_frame_crop_results,
+    const int scene_frame_width, const int scene_frame_height,
+    const std::vector<int64>& scene_frame_timestamps,
+    SceneKeyFrameCropSummary* scene_summary,
+    std::vector<FocusPointFrame>* focus_point_frames,
+    SceneCameraMotion* scene_camera_motion) const {
+  MP_RETURN_IF_ERROR(AggregateKeyFrameResults(
+      key_frame_infos, key_frame_crop_options, key_frame_crop_results,
+      scene_frame_width, scene_frame_height, scene_summary));
+
+  const int64 scene_span_ms =
+      scene_frame_timestamps.empty()
+          ? 0
+          : scene_frame_timestamps.back() - scene_frame_timestamps.front();
+  const double scene_span_sec = TimestampDiff(scene_span_ms).Seconds();
+  SceneCameraMotion camera_motion;
+  MP_RETURN_IF_ERROR(DecideCameraMotionType(
+      key_frame_crop_options, scene_span_sec, scene_summary, &camera_motion));
+  if (scene_camera_motion != nullptr) {
+    *scene_camera_motion = camera_motion;
+  }
+
+  return PopulateFocusPointFrames(*scene_summary, camera_motion,
+                                  scene_frame_timestamps, focus_point_frames);
+}
+
+::mediapipe::Status SceneCameraMotionAnalyzer::ToUseSteadyMotion(
+    const float look_at_center_x, const float look_at_center_y,
+    const int crop_window_width, const int crop_window_height,
+    SceneKeyFrameCropSummary* scene_summary,
+    SceneCameraMotion* scene_camera_motion) const {
+  scene_summary->set_crop_window_width(crop_window_width);
+  scene_summary->set_crop_window_height(crop_window_height);
+  auto* steady_motion = scene_camera_motion->mutable_steady_motion();
+  steady_motion->set_steady_look_at_center_x(look_at_center_x);
+  steady_motion->set_steady_look_at_center_y(look_at_center_y);
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status SceneCameraMotionAnalyzer::ToUseSweepingMotion(
+    const float start_x, const float start_y, const float end_x,
+    const float end_y, const int crop_window_width,
+    const int crop_window_height, const double time_duration_in_sec,
+    SceneKeyFrameCropSummary* scene_summary,
+    SceneCameraMotion* scene_camera_motion) const {
+  auto* sweeping_motion = scene_camera_motion->mutable_sweeping_motion();
+  sweeping_motion->set_sweep_start_center_x(start_x);
+  sweeping_motion->set_sweep_start_center_y(start_y);
+  sweeping_motion->set_sweep_end_center_x(end_x);
+  sweeping_motion->set_sweep_end_center_y(end_y);
+  scene_summary->set_crop_window_width(crop_window_width);
+  scene_summary->set_crop_window_height(crop_window_height);
+  const auto sweeping_log = absl::StrFormat(
+      "Success rate %.2f is low - Camera is sweeping from (%.1f, %.1f) to "
+      "(%.1f, %.1f) in %.2f seconds.",
+      scene_summary->frame_success_rate(), start_x, start_y, end_x, end_y,
+      time_duration_in_sec);
+  VLOG(1) << sweeping_log;
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status SceneCameraMotionAnalyzer::DecideCameraMotionType(
+    const KeyFrameCropOptions& key_frame_crop_options,
+    const double scene_span_sec, SceneKeyFrameCropSummary* scene_summary,
+    SceneCameraMotion* scene_camera_motion) const {
+  RET_CHECK_GE(scene_span_sec, 0.0) << "Scene time span is negative.";
+  RET_CHECK_NE(scene_summary, nullptr) << "Scene summary is null.";
+  RET_CHECK_NE(scene_camera_motion, nullptr) << "Scene camera motion is null.";
+  const float scene_frame_center_x = scene_summary->scene_frame_width() / 2.0f;
+  const float scene_frame_center_y = scene_summary->scene_frame_height() / 2.0f;
+
+  // If no frame has any focus region, that is, the scene has no focus
+  // regions, then default to look at the center.
+  if (!scene_summary->has_salient_region()) {
+    VLOG(1) << "No focus regions - camera is set to be steady on center.";
+    MP_RETURN_IF_ERROR(ToUseSteadyMotion(
+        scene_frame_center_x, scene_frame_center_y,
+        scene_summary->crop_window_width(), scene_summary->crop_window_height(),
+        scene_summary, scene_camera_motion));
+    return ::mediapipe::OkStatus();
+  }
+
+  // Sweep across the scene when 1) success rate is too low, AND 2) the current
+  // scene is long enough.
+  if (options_.allow_sweeping() &&
+      scene_summary->frame_success_rate() <
+          options_.minimum_success_rate_for_sweeping() &&
+      scene_span_sec >= options_.minimum_scene_span_sec_for_sweeping()) {
+    float start_x = -1.0, start_y = -1.0, end_x = -1.0, end_y = -1.0;
+    if (options_.sweep_entire_frame()) {
+      if (scene_summary->crop_window_width() >
+          key_frame_crop_options.target_width()) {  // horizontal sweeping
+        start_x = 0.0f;
+        start_y = scene_frame_center_y;
+        end_x = scene_summary->scene_frame_width();
+        end_y = scene_frame_center_y;
+      } else {  // vertical sweeping
+        start_x = scene_frame_center_x;
+        start_y = 0.0f;
+        end_x = scene_frame_center_x;
+        end_y = scene_summary->scene_frame_height();
+      }
+    } else {
+      start_x = scene_summary->key_frame_center_min_x();
+      start_y = scene_summary->key_frame_center_min_y();
+      end_x = scene_summary->key_frame_center_max_x();
+      end_y = scene_summary->key_frame_center_max_y();
+    }
+    MP_RETURN_IF_ERROR(ToUseSweepingMotion(
+        start_x, start_y, end_x, end_y, key_frame_crop_options.target_width(),
+        key_frame_crop_options.target_height(), scene_span_sec, scene_summary,
+        scene_camera_motion));
+    return ::mediapipe::OkStatus();
+  }
+
+  // If scene motion is small, then look at a steady point in the scene.
+  if (scene_summary->horizontal_motion_amount() <
+          options_.motion_stabilization_threshold_percent() &&
+      scene_summary->vertical_motion_amount() <
+          options_.motion_stabilization_threshold_percent()) {
+    return DecideSteadyLookAtRegion(key_frame_crop_options, scene_summary,
+                                    scene_camera_motion);
+  }
+
+  // Otherwise, tracks the focus regions.
+  scene_camera_motion->mutable_tracking_motion();
+  return ::mediapipe::OkStatus();
+}
+
+// If there is no required focus region, looks at the middle of the center
+// range, and snaps to the scene center if close. Otherwise, look at the center
+// of the union of the required focus regions, and ensures the crop region
+// covers this union.
+::mediapipe::Status SceneCameraMotionAnalyzer::DecideSteadyLookAtRegion(
+    const KeyFrameCropOptions& key_frame_crop_options,
+    SceneKeyFrameCropSummary* scene_summary,
+    SceneCameraMotion* scene_camera_motion) const {
+  const float scene_frame_width = scene_summary->scene_frame_width();
+  const float scene_frame_height = scene_summary->scene_frame_height();
+  const int target_width = key_frame_crop_options.target_width();
+  const int target_height = key_frame_crop_options.target_height();
+  float center_x = -1, center_y = -1;
+  float crop_width = -1, crop_height = -1;
+
+  if (scene_summary->has_required_salient_region()) {
+    // Set look-at position to be the center of the union of required focus
+    // regions and the crop window size to be the maximum of this union size
+    // and the target size.
+    const auto& required_region_union =
+        scene_summary->key_frame_required_crop_region_union();
+    center_x = required_region_union.x() + required_region_union.width() / 2.0f;
+    center_y =
+        required_region_union.y() + required_region_union.height() / 2.0f;
+    crop_width = std::max(target_width, required_region_union.width());
+    crop_height = std::max(target_height, required_region_union.height());
+  } else {
+    // Set look-at position to be the middle of the center range, and the crop
+    // window size to be the target size.
+    center_x = (scene_summary->key_frame_center_min_x() +
+                scene_summary->key_frame_center_max_x()) /
+               2.0f;
+    center_y = (scene_summary->key_frame_center_min_y() +
+                scene_summary->key_frame_center_max_y()) /
+               2.0f;
+    crop_width = target_width;
+    crop_height = target_height;
+
+    // Optionally snap the look-at position to the scene frame center.
+    const float center_x_distance =
+        std::fabs(center_x - scene_frame_width / 2.0f);
+    const float center_y_distance =
+        std::fabs(center_y - scene_frame_height / 2.0f);
+    if (center_x_distance / scene_frame_width <
+        options_.snap_center_max_distance_percent()) {
+      center_x = scene_frame_width / 2.0f;
+    }
+    if (center_y_distance / scene_frame_height <
+        options_.snap_center_max_distance_percent()) {
+      center_y = scene_frame_height / 2.0f;
+    }
+  }
+
+  // Clamp the region to be inside the frame.
+  // TODO: this may not be necessary.
+  float clamped_center_x, clamped_center_y;
+  RET_CHECK(MathUtil::Clamp(crop_width / 2.0f,
+                            scene_frame_width - crop_width / 2.0f, center_x,
+                            &clamped_center_x));
+  center_x = clamped_center_x;
+  RET_CHECK(MathUtil::Clamp(crop_height / 2.0f,
+                            scene_frame_height - crop_height / 2.0f, center_y,
+                            &clamped_center_y));
+  center_y = clamped_center_y;
+
+  VLOG(1) << "Motion is small - camera is set to be steady at " << center_x
+          << ", " << center_y;
+  MP_RETURN_IF_ERROR(ToUseSteadyMotion(center_x, center_y, crop_width,
+                                       crop_height, scene_summary,
+                                       scene_camera_motion));
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status
+SceneCameraMotionAnalyzer::AddFocusPointsFromCenterTypeAndWeight(
+    const float center_x, const float center_y, const int frame_width,
+    const int frame_height, const FocusPointFrameType type, const float weight,
+    const float bound, FocusPointFrame* focus_point_frame) const {
+  RET_CHECK_NE(focus_point_frame, nullptr) << "Focus point frame is null.";
+  const float norm_x = center_x / frame_width;
+  const float norm_y = center_y / frame_height;
+  const std::vector<float> extremal_values = {0, 1};
+  if (type == TOPMOST_AND_BOTTOMMOST) {
+    for (const float extremal_value : extremal_values) {
+      auto* focus_point = focus_point_frame->add_point();
+      focus_point->set_norm_point_x(norm_x);
+      focus_point->set_norm_point_y(extremal_value);
+      focus_point->set_weight(weight);
+      focus_point->set_left(bound);
+      focus_point->set_right(bound);
+    }
+  } else if (type == LEFTMOST_AND_RIGHTMOST) {
+    for (const float extremal_value : extremal_values) {
+      auto* focus_point = focus_point_frame->add_point();
+      focus_point->set_norm_point_x(extremal_value);
+      focus_point->set_norm_point_y(norm_y);
+      focus_point->set_weight(weight);
+      focus_point->set_top(bound);
+      focus_point->set_bottom(bound);
+    }
+  } else if (type == CENTER) {
+    auto* focus_point = focus_point_frame->add_point();
+    focus_point->set_norm_point_x(norm_x);
+    focus_point->set_norm_point_y(norm_y);
+    focus_point->set_weight(weight);
+    focus_point->set_left(bound);
+    focus_point->set_right(bound);
+    focus_point->set_top(bound);
+    focus_point->set_bottom(bound);
+  } else {
+    RET_CHECK_FAIL() << absl::StrCat("Invalid FocusPointFrameType ", type);
+  }
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status SceneCameraMotionAnalyzer::PopulateFocusPointFrames(
+    const SceneKeyFrameCropSummary& scene_summary,
+    const SceneCameraMotion& scene_camera_motion,
+    const std::vector<int64>& scene_frame_timestamps,
+    std::vector<FocusPointFrame>* focus_point_frames) const {
+  RET_CHECK_NE(focus_point_frames, nullptr)
+      << "Output vector of FocusPointFrame is null.";
+
+  const int num_scene_frames = scene_frame_timestamps.size();
+  RET_CHECK_GT(num_scene_frames, 0) << "No scene frames.";
+  RET_CHECK_EQ(scene_summary.num_key_frames(),
+               scene_summary.key_frame_compact_infos_size())
+      << "Key frame compact infos has wrong size:"
+      << " num_key_frames = " << scene_summary.num_key_frames()
+      << " key_frame_compact_infos size = "
+      << scene_summary.key_frame_compact_infos_size();
+  const int scene_frame_width = scene_summary.scene_frame_width();
+  const int scene_frame_height = scene_summary.scene_frame_height();
+  RET_CHECK_GT(scene_frame_width, 0) << "Non-positive frame width.";
+  RET_CHECK_GT(scene_frame_height, 0) << "Non-positive frame height.";
+
+  FocusPointFrameType focus_point_frame_type =
+      (scene_summary.crop_window_height() == scene_frame_height)
+          ? TOPMOST_AND_BOTTOMMOST
+          : (scene_summary.crop_window_width() == scene_frame_width
+                 ? LEFTMOST_AND_RIGHTMOST
+                 : CENTER);
+  focus_point_frames->reserve(num_scene_frames);
+
+  if (scene_camera_motion.has_steady_motion()) {
+    // Camera focuses on a steady point of the scene.
+    const float center_x =
+        scene_camera_motion.steady_motion().steady_look_at_center_x();
+    const float center_y =
+        scene_camera_motion.steady_motion().steady_look_at_center_y();
+    for (int i = 0; i < num_scene_frames; ++i) {
+      FocusPointFrame focus_point_frame;
+      MP_RETURN_IF_ERROR(AddFocusPointsFromCenterTypeAndWeight(
+          center_x, center_y, scene_frame_width, scene_frame_height,
+          focus_point_frame_type, options_.maximum_salient_point_weight(),
+          options_.salient_point_bound(), &focus_point_frame));
+      focus_point_frames->push_back(focus_point_frame);
+    }
+    return ::mediapipe::OkStatus();
+  } else if (scene_camera_motion.has_sweeping_motion()) {
+    // Camera sweeps across the frame.
+    const auto& sweeping_motion = scene_camera_motion.sweeping_motion();
+    const float start_x = sweeping_motion.sweep_start_center_x();
+    const float start_y = sweeping_motion.sweep_start_center_y();
+    const float end_x = sweeping_motion.sweep_end_center_x();
+    const float end_y = sweeping_motion.sweep_end_center_y();
+    for (int i = 0; i < num_scene_frames; ++i) {
+      const float fraction =
+          num_scene_frames > 1 ? static_cast<float>(i) / (num_scene_frames - 1)
+                               : 0;
+      const float position_x = start_x * (1.0f - fraction) + end_x * fraction;
+      const float position_y = start_y * (1.0f - fraction) + end_y * fraction;
+      FocusPointFrame focus_point_frame;
+      MP_RETURN_IF_ERROR(AddFocusPointsFromCenterTypeAndWeight(
+          position_x, position_y, scene_frame_width, scene_frame_height,
+          focus_point_frame_type, options_.maximum_salient_point_weight(),
+          options_.salient_point_bound(), &focus_point_frame));
+      focus_point_frames->push_back(focus_point_frame);
+    }
+    return ::mediapipe::OkStatus();
+  } else if (scene_camera_motion.has_tracking_motion()) {
+    // Camera tracks crop regions.
+    RET_CHECK_GT(scene_summary.num_key_frames(), 0) << "No key frames.";
+    return PopulateFocusPointFramesForTracking(
+        scene_summary, focus_point_frame_type, scene_frame_timestamps,
+        focus_point_frames);
+  } else {
+    return ::mediapipe::Status(StatusCode::kInvalidArgument,
+                               "Unknown motion type.");
+  }
+}
+
+// Linearly interpolates between key frames based on the timestamps using
+// piecewise-linear functions for the crop region centers and scores. Adds one
+// focus point at the center of the interpolated crop region for each frame.
+// The weight for the focus point is proportional to the interpolated score
+// and scaled so that the maximum weight is equal to
+// maximum_focus_point_weight in the SceneCameraMotionAnalyzerOptions.
+::mediapipe::Status
+SceneCameraMotionAnalyzer::PopulateFocusPointFramesForTracking(
+    const SceneKeyFrameCropSummary& scene_summary,
+    const FocusPointFrameType focus_point_frame_type,
+    const std::vector<int64>& scene_frame_timestamps,
+    std::vector<FocusPointFrame>* focus_point_frames) const {
+  RET_CHECK_GE(scene_summary.key_frame_max_score(), 0.0)
+      << "Maximum score is negative.";
+
+  const int num_key_frames = scene_summary.num_key_frames();
+  const auto& key_frame_compact_infos = scene_summary.key_frame_compact_infos();
+  const int num_scene_frames = scene_frame_timestamps.size();
+  const int scene_frame_width = scene_summary.scene_frame_width();
+  const int scene_frame_height = scene_summary.scene_frame_height();
+
+  PiecewiseLinearFunction center_x_function, center_y_function, score_function;
+  const int64 timestamp_offset = key_frame_compact_infos[0].timestamp_ms();
+  for (int i = 0; i < num_key_frames; ++i) {
+    const float center_x = key_frame_compact_infos[i].center_x();
+    const float center_y = key_frame_compact_infos[i].center_y();
+    const float score = key_frame_compact_infos[i].score();
+    // Skips empty key frames.
+    if (center_x < 0 || center_y < 0 || score < 0) {
+      continue;
+    }
+    const double relative_timestamp =
+        key_frame_compact_infos[i].timestamp_ms() - timestamp_offset;
+    center_x_function.AddPoint(relative_timestamp, center_x);
+    center_y_function.AddPoint(relative_timestamp, center_y);
+    score_function.AddPoint(relative_timestamp, score);
+  }
+
+  double max_score = 0.0;
+  const double min_score = 1e-4;  // prevent constraints with 0 weight
+  for (int i = 0; i < num_scene_frames; ++i) {
+    const double relative_timestamp =
+        static_cast<double>(scene_frame_timestamps[i] - timestamp_offset);
+    const double center_x = center_x_function.Evaluate(relative_timestamp);
+    const double center_y = center_y_function.Evaluate(relative_timestamp);
+    const double score =
+        std::max(min_score, score_function.Evaluate(relative_timestamp));
+    max_score = std::max(max_score, score);
+    FocusPointFrame focus_point_frame;
+    MP_RETURN_IF_ERROR(AddFocusPointsFromCenterTypeAndWeight(
+        center_x, center_y, scene_frame_width, scene_frame_height,
+        focus_point_frame_type, score, options_.salient_point_bound(),
+        &focus_point_frame));
+    focus_point_frames->push_back(focus_point_frame);
+  }
+
+  // Scales weights so that maximum weight = maximum_salient_point_weight.
+  // TODO: run some experiments to find out if this is necessary.
+  max_score = std::max(max_score, min_score);
+  const double scale = options_.maximum_salient_point_weight() / max_score;
+  for (int i = 0; i < focus_point_frames->size(); ++i) {
+    for (int j = 0; j < (*focus_point_frames)[i].point_size(); ++j) {
+      auto* focus_point = (*focus_point_frames)[i].mutable_point(j);
+      focus_point->set_weight(scale * focus_point->weight());
+    }
+  }
+  return ::mediapipe::OkStatus();
+}
+
+}  // namespace autoflip
+}  // namespace mediapipe

mediapipe/examples/desktop/autoflip/quality/scene_camera_motion_analyzer.h

@@ -0,0 +1,142 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_SCENE_CAMERA_MOTION_ANALYZER_H_
+#define MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_SCENE_CAMERA_MOTION_ANALYZER_H_
+
+#include <vector>
+
+#include "mediapipe/examples/desktop/autoflip/autoflip_messages.pb.h"
+#include "mediapipe/examples/desktop/autoflip/quality/cropping.pb.h"
+#include "mediapipe/examples/desktop/autoflip/quality/focus_point.pb.h"
+#include "mediapipe/framework/port/integral_types.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+namespace autoflip {
+
+// This class does the following in order:
+// - Aggregates a key frame results to get a SceneKeyFrameCropSummary,
+// - Determines the SceneCameraMotion for the scene, and then
+// - Populates FocusPointFrames to be used as input for the retargeter.
+//
+// Upstream inputs:
+// - std::vector<KeyFrameCropInfo> key_frame_crop_infos.
+// - KeyFrameCropOptions key_frame_crop_options.
+// - std::vector<KeyFrameCropResult> key_frame_crop_results.
+// - int scene_frame_width, scene_frame_height.
+// - std::vector<int64> scene_frame_timestamps.
+//
+// Example usage:
+//   SceneCameraMotionAnalyzerOptions options;
+//   SceneCameraMotionAnalyzer analyzer(options);
+//   SceneKeyFrameCropSummary scene_summary;
+//   std::vector<FocusPointFrame> focus_point_frames;
+//   CHECK_OK(analyzer.AnalyzeScenePopulateFocusPointFrames(
+//       key_frame_crop_infos, key_frame_crop_options, key_frame_crop_results,
+//       scene_frame_width, scene_frame_height, scene_frame_timestamps,
+//       &scene_summary, &focus_point_frames));
+class SceneCameraMotionAnalyzer {
+ public:
+  SceneCameraMotionAnalyzer() = delete;
+
+  explicit SceneCameraMotionAnalyzer(const SceneCameraMotionAnalyzerOptions&
+                                         scene_camera_motion_analyzer_options)
+      : options_(scene_camera_motion_analyzer_options) {}
+
+  ~SceneCameraMotionAnalyzer() {}
+
+  // Aggregates information from KeyFrameInfos and KeyFrameCropResults into
+  // SceneKeyFrameCropSummary, and populates FocusPointFrames given scene
+  // frame timestamps. Optionally returns SceneCameraMotion.
+  ::mediapipe::Status AnalyzeSceneAndPopulateFocusPointFrames(
+      const std::vector<KeyFrameInfo>& key_frame_infos,
+      const KeyFrameCropOptions& key_frame_crop_options,
+      const std::vector<KeyFrameCropResult>& key_frame_crop_results,
+      const int scene_frame_width, const int scene_frame_height,
+      const std::vector<int64>& scene_frame_timestamps,
+      SceneKeyFrameCropSummary* scene_summary,
+      std::vector<FocusPointFrame>* focus_point_frames,
+      SceneCameraMotion* scene_camera_motion = nullptr) const;
+
+ protected:
+  // Decides SceneCameraMotion based on SceneKeyFrameCropSummary. Updates the
+  // crop window in SceneKeyFrameCropSummary in the case of steady motion.
+  ::mediapipe::Status DecideCameraMotionType(
+      const KeyFrameCropOptions& key_frame_crop_options,
+      const double scene_span_sec, SceneKeyFrameCropSummary* scene_summary,
+      SceneCameraMotion* scene_camera_motion) const;
+
+  // Populates the FocusPointFrames for each scene frame based on
+  // SceneKeyFrameCropSummary, SceneCameraMotion, and scene frame timestamps.
+  ::mediapipe::Status PopulateFocusPointFrames(
+      const SceneKeyFrameCropSummary& scene_summary,
+      const SceneCameraMotion& scene_camera_motion,
+      const std::vector<int64>& scene_frame_timestamps,
+      std::vector<FocusPointFrame>* focus_point_frames) const;
+
+ private:
+  // Decides the look-at region when camera is steady.
+  ::mediapipe::Status DecideSteadyLookAtRegion(
+      const KeyFrameCropOptions& key_frame_crop_options,
+      SceneKeyFrameCropSummary* scene_summary,
+      SceneCameraMotion* scene_camera_motion) const;
+
+  // Types of FocusPointFrames: number and placement of FocusPoint's vary.
+  enum FocusPointFrameType {
+    TOPMOST_AND_BOTTOMMOST = 1,  // (center_x, 0) and (center_x, frame_height)
+    LEFTMOST_AND_RIGHTMOST = 2,  // (0, center_y) and (frame_width, center_y)
+    CENTER = 3,                  // (center_x, center_y)
+  };
+
+  // Adds FocusPoint(s) to given FocusPointFrame given center location,
+  // frame size, FocusPointFrameType, weight, and bound.
+  ::mediapipe::Status AddFocusPointsFromCenterTypeAndWeight(
+      const float center_x, const float center_y, const int frame_width,
+      const int frame_height, const FocusPointFrameType type,
+      const float weight, const float bound,
+      FocusPointFrame* focus_point_frame) const;
+
+  // Populates the FocusPointFrames for each scene frame based on
+  // SceneKeyFrameCropSummary and scene frame timestamps in the case where
+  // camera is tracking the crop regions.
+  ::mediapipe::Status PopulateFocusPointFramesForTracking(
+      const SceneKeyFrameCropSummary& scene_summary,
+      const FocusPointFrameType focus_point_frame_type,
+      const std::vector<int64>& scene_frame_timestamps,
+      std::vector<FocusPointFrame>* focus_point_frames) const;
+
+  // Decide to use steady motion.
+  ::mediapipe::Status ToUseSteadyMotion(
+      const float look_at_center_x, const float look_at_center_y,
+      const int crop_window_width, const int crop_window_height,
+      SceneKeyFrameCropSummary* scene_summary,
+      SceneCameraMotion* scene_camera_motion) const;
+
+  // Decide to use sweeping motion.
+  ::mediapipe::Status ToUseSweepingMotion(
+      const float start_x, const float start_y, const float end_x,
+      const float end_y, const int crop_window_width,
+      const int crop_window_height, const double time_duration_in_sec,
+      SceneKeyFrameCropSummary* scene_summary,
+      SceneCameraMotion* scene_camera_motion) const;
+
+  // Scene camera motion analyzer options.
+  SceneCameraMotionAnalyzerOptions options_;
+};
+
+}  // namespace autoflip
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_SCENE_CAMERA_MOTION_ANALYZER_H_

mediapipe/examples/desktop/autoflip/quality/scene_camera_motion_analyzer_test.cc

@@ -0,0 +1,814 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/examples/desktop/autoflip/quality/scene_camera_motion_analyzer.h"
+
+#include <algorithm>
+#include <numeric>
+#include <string>
+#include <vector>
+
+#include "absl/strings/str_split.h"
+#include "mediapipe/examples/desktop/autoflip/autoflip_messages.pb.h"
+#include "mediapipe/examples/desktop/autoflip/quality/focus_point.pb.h"
+#include "mediapipe/examples/desktop/autoflip/quality/piecewise_linear_function.h"
+#include "mediapipe/framework/deps/file_path.h"
+#include "mediapipe/framework/port/file_helpers.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/port/status_matchers.h"
+
+namespace mediapipe {
+namespace autoflip {
+
+using ::testing::HasSubstr;
+
+const int kNumKeyFrames = 5;
+const int kNumSceneFrames = 30;
+
+const int64 kKeyFrameTimestampDiff = 1e6 / kNumKeyFrames;
+const int64 kSceneFrameTimestampDiff = 1e6 / kNumSceneFrames;
+// Default time span of a scene in seconds.
+const double kSceneTimeSpanSec = 1.0;
+
+const int kSceneFrameWidth = 100;
+const int kSceneFrameHeight = 100;
+
+const int kTargetWidth = 50;
+const int kTargetHeight = 50;
+
+constexpr char kCameraTrackingSceneFrameResultsFile[] =
+    "mediapipe/examples/desktop/autoflip/quality/testdata/"
+    "camera_motion_tracking_scene_frame_results.csv";
+
+// Makes a rectangle given the corner (x, y) and the size (width, height).
+Rect MakeRect(const int x, const int y, const int width, const int height) {
+  Rect rect;
+  rect.set_x(x);
+  rect.set_y(y);
+  rect.set_width(width);
+  rect.set_height(height);
+  return rect;
+}
+
+// Returns default values for KeyFrameInfos. Populates timestamps using the
+// default spacing kKeyFrameTimestampDiff starting from 0.
+std::vector<KeyFrameInfo> GetDefaultKeyFrameInfos() {
+  std::vector<KeyFrameInfo> key_frame_infos(kNumKeyFrames);
+  for (int i = 0; i < kNumKeyFrames; ++i) {
+    key_frame_infos[i].set_timestamp_ms(kKeyFrameTimestampDiff * i);
+  }
+  return key_frame_infos;
+}
+
+// Returns default values for scene frame timestamps. Populates timestamps using
+// the default spacing kSceneFrameTimestampDiff starting from 0.
+std::vector<int64> GetDefaultSceneFrameTimestamps() {
+  std::vector<int64> scene_frame_timestamps(kNumSceneFrames);
+  for (int i = 0; i < kNumSceneFrames; ++i) {
+    scene_frame_timestamps[i] = kSceneFrameTimestampDiff * i;
+  }
+  return scene_frame_timestamps;
+}
+
+// Returns default settings for KeyFrameCropOptions. Populates target size to be
+// the default target size.
+KeyFrameCropOptions GetDefaultKeyFrameCropOptions() {
+  KeyFrameCropOptions key_frame_crop_options;
+  key_frame_crop_options.set_target_width(kTargetWidth);
+  key_frame_crop_options.set_target_height(kTargetHeight);
+  return key_frame_crop_options;
+}
+
+// Returns default values for KeyFrameCropResults. Sets each frame to have
+// covered all the required regions and non-required regions, and have required
+// crop region (10, 10+20) x (10, 10+20), (full) crop region (0, 50) x (0, 50),
+// and region score 1.0.
+std::vector<KeyFrameCropResult> GetDefaultKeyFrameCropResults() {
+  std::vector<KeyFrameCropResult> key_frame_crop_results(kNumKeyFrames);
+  for (int i = 0; i < kNumKeyFrames; ++i) {
+    key_frame_crop_results[i].set_are_required_regions_covered_in_target_size(
+        true);
+    key_frame_crop_results[i].set_fraction_non_required_covered(1.0);
+    key_frame_crop_results[i].set_region_is_empty(false);
+    key_frame_crop_results[i].set_required_region_is_empty(false);
+    *(key_frame_crop_results[i].mutable_region()) = MakeRect(0, 0, 50, 50);
+    *(key_frame_crop_results[i].mutable_required_region()) =
+        MakeRect(10, 10, 20, 20);
+    key_frame_crop_results[i].set_region_score(1.0);
+  }
+  return key_frame_crop_results;
+}
+
+// Returns default settings for SceneKeyFrameCropSummary. Sets scene frame size
+// to be the default size. Sets each key frame compact info in accordance to the
+// default timestamps (using the default spacing kKeyFrameTimestampDiff starting
+// from 0), default crop regions (centered at (25, 25)), and default scores
+// (1.0). Sets center range to be [25, 25] and [25, 25]. Sets score range to be
+// [1.0, 1.0]. Sets crop window size to be (25, 25). Sets has focus region to
+// be true. Sets frame success rate to be 1.0. Sets horizontal and vertical
+// motion amount to be 0.0.
+SceneKeyFrameCropSummary GetDefaultSceneKeyFrameCropSummary() {
+  SceneKeyFrameCropSummary scene_summary;
+  scene_summary.set_scene_frame_width(kSceneFrameWidth);
+  scene_summary.set_scene_frame_height(kSceneFrameHeight);
+  scene_summary.set_num_key_frames(kNumKeyFrames);
+  for (int i = 0; i < kNumKeyFrames; ++i) {
+    auto* compact_info = scene_summary.add_key_frame_compact_infos();
+    compact_info->set_timestamp_ms(kKeyFrameTimestampDiff * i);
+    compact_info->set_center_x(25);
+    compact_info->set_center_y(25);
+    compact_info->set_score(1.0);
+  }
+  scene_summary.set_key_frame_center_min_x(25);
+  scene_summary.set_key_frame_center_max_x(25);
+  scene_summary.set_key_frame_center_min_y(25);
+  scene_summary.set_key_frame_center_max_y(25);
+  scene_summary.set_key_frame_min_score(1.0);
+  scene_summary.set_key_frame_max_score(1.0);
+  scene_summary.set_crop_window_width(25);
+  scene_summary.set_crop_window_height(25);
+  scene_summary.set_has_salient_region(true);
+  scene_summary.set_frame_success_rate(1.0);
+  scene_summary.set_horizontal_motion_amount(0.0);
+  scene_summary.set_vertical_motion_amount(0.0);
+  return scene_summary;
+}
+
+// Returns a SceneKeyFrameCropSummary with small motion. Sets crop window size
+// to default target size. Sets horizontal motion to half the threshold in the
+// options and vertical motion to 0. Sets center x range to [45, 55].
+SceneKeyFrameCropSummary GetSceneKeyFrameCropSummaryWithSmallMotion(
+    const SceneCameraMotionAnalyzerOptions& options) {
+  auto scene_summary = GetDefaultSceneKeyFrameCropSummary();
+  scene_summary.set_crop_window_width(kTargetWidth);
+  scene_summary.set_crop_window_height(kTargetHeight);
+  scene_summary.set_horizontal_motion_amount(
+      options.motion_stabilization_threshold_percent() / 2.0);
+  scene_summary.set_vertical_motion_amount(0.0);
+  scene_summary.set_key_frame_center_min_x(45);
+  scene_summary.set_key_frame_center_max_x(55);
+  return scene_summary;
+}
+
+// Testable class that allows public access to protected methods in the class.
+class TestableSceneCameraMotionAnalyzer : public SceneCameraMotionAnalyzer {
+ public:
+  explicit TestableSceneCameraMotionAnalyzer(
+      const SceneCameraMotionAnalyzerOptions&
+          scene_camera_motion_analyzer_options)
+      : SceneCameraMotionAnalyzer(scene_camera_motion_analyzer_options) {}
+  ~TestableSceneCameraMotionAnalyzer() {}
+  using SceneCameraMotionAnalyzer::DecideCameraMotionType;
+  using SceneCameraMotionAnalyzer::PopulateFocusPointFrames;
+};
+
+// Checks that DecideCameraMotionType checks that output pointers are not null.
+TEST(SceneCameraMotionAnalyzerTest, DecideCameraMotionTypeChecksOutputNotNull) {
+  SceneCameraMotionAnalyzerOptions options;
+  TestableSceneCameraMotionAnalyzer analyzer(options);
+  KeyFrameCropOptions crop_options = GetDefaultKeyFrameCropOptions();
+  SceneKeyFrameCropSummary scene_summary;
+  SceneCameraMotion camera_motion;
+  auto status = analyzer.DecideCameraMotionType(crop_options, kSceneTimeSpanSec,
+                                                nullptr, &camera_motion);
+  EXPECT_FALSE(status.ok());
+  EXPECT_THAT(status.ToString(), HasSubstr("Scene summary is null."));
+  status = analyzer.DecideCameraMotionType(crop_options, kSceneTimeSpanSec,
+                                           &scene_summary, nullptr);
+  EXPECT_FALSE(status.ok());
+  EXPECT_THAT(status.ToString(), HasSubstr("Scene camera motion is null."));
+}
+
+// Checks that DecideCameraMotionType properly handles the case where no key
+// frame has any focus region, and sets the camera motion type to steady and
+// the look-at position to the scene frame center.
+TEST(SceneCameraMotionAnalyzerTest,
+     DecideCameraMotionTypeWithoutAnyFocusRegion) {
+  SceneCameraMotionAnalyzerOptions options;
+  TestableSceneCameraMotionAnalyzer analyzer(options);
+  KeyFrameCropOptions crop_options = GetDefaultKeyFrameCropOptions();
+  auto scene_summary = GetDefaultSceneKeyFrameCropSummary();
+  scene_summary.set_has_salient_region(false);
+  SceneCameraMotion camera_motion;
+
+  MP_EXPECT_OK(analyzer.DecideCameraMotionType(crop_options, kSceneTimeSpanSec,
+                                               &scene_summary, &camera_motion));
+  EXPECT_TRUE(camera_motion.has_steady_motion());
+  const auto& steady_motion = camera_motion.steady_motion();
+  EXPECT_FLOAT_EQ(steady_motion.steady_look_at_center_x(),
+                  kSceneFrameWidth / 2.0f);
+  EXPECT_FLOAT_EQ(steady_motion.steady_look_at_center_y(),
+                  kSceneFrameHeight / 2.0f);
+}
+
+// Checks that DecideCameraMotionType properly handles the camera sweeps from
+// left to right.
+TEST(SceneCameraMotionAnalyzerTest, DecideCameraMotionTypeSweepingLeftToRight) {
+  SceneCameraMotionAnalyzerOptions options;
+  options.set_sweep_entire_frame(true);
+  TestableSceneCameraMotionAnalyzer analyzer(options);
+  auto scene_summary = GetDefaultSceneKeyFrameCropSummary();
+  scene_summary.set_frame_success_rate(
+      options.minimum_success_rate_for_sweeping() / 2.0f);
+  scene_summary.set_crop_window_width(kTargetWidth * 1.5f);  // horizontal sweep
+  scene_summary.set_crop_window_height(kTargetHeight);
+  const double time_span = options.minimum_scene_span_sec_for_sweeping() * 2.0;
+  SceneCameraMotion camera_motion;
+
+  MP_EXPECT_OK(analyzer.DecideCameraMotionType(GetDefaultKeyFrameCropOptions(),
+                                               time_span, &scene_summary,
+                                               &camera_motion));
+
+  EXPECT_TRUE(camera_motion.has_sweeping_motion());
+  const auto& sweeping_motion = camera_motion.sweeping_motion();
+  EXPECT_FLOAT_EQ(sweeping_motion.sweep_start_center_x(), 0.0f);
+  EXPECT_FLOAT_EQ(sweeping_motion.sweep_end_center_x(),
+                  static_cast<float>(kSceneFrameWidth));
+  EXPECT_FLOAT_EQ(sweeping_motion.sweep_start_center_y(),
+                  kSceneFrameHeight / 2.0f);
+  EXPECT_FLOAT_EQ(sweeping_motion.sweep_end_center_y(),
+                  kSceneFrameHeight / 2.0f);
+}
+
+// Checks that DecideCameraMotionType properly handles the camera sweeps from
+// top to bottom.
+TEST(SceneCameraMotionAnalyzerTest, DecideCameraMotionTypeSweepingTopToBottom) {
+  SceneCameraMotionAnalyzerOptions options;
+  options.set_sweep_entire_frame(true);
+  TestableSceneCameraMotionAnalyzer analyzer(options);
+  auto scene_summary = GetDefaultSceneKeyFrameCropSummary();
+  scene_summary.set_frame_success_rate(
+      options.minimum_success_rate_for_sweeping() / 2.0f);
+  scene_summary.set_crop_window_width(kTargetWidth);
+  scene_summary.set_crop_window_height(kTargetHeight * 1.5f);  // vertical sweep
+  const double time_span = options.minimum_scene_span_sec_for_sweeping() * 2.0;
+  SceneCameraMotion camera_motion;
+
+  MP_EXPECT_OK(analyzer.DecideCameraMotionType(GetDefaultKeyFrameCropOptions(),
+                                               time_span, &scene_summary,
+                                               &camera_motion));
+
+  EXPECT_TRUE(camera_motion.has_sweeping_motion());
+  const auto& sweeping_motion = camera_motion.sweeping_motion();
+  EXPECT_FLOAT_EQ(sweeping_motion.sweep_start_center_y(), 0.0f);
+  EXPECT_FLOAT_EQ(sweeping_motion.sweep_end_center_y(),
+                  static_cast<float>(kSceneFrameWidth));
+  EXPECT_FLOAT_EQ(sweeping_motion.sweep_start_center_x(),
+                  kSceneFrameWidth / 2.0f);
+  EXPECT_FLOAT_EQ(sweeping_motion.sweep_end_center_x(),
+                  kSceneFrameWidth / 2.0f);
+}
+
+// Checks that DecideCameraMotionType properly handles the camera sweeps from
+// one corner of the center range to another.
+TEST(SceneCameraMotionAnalyzerTest, DecideCameraMotionTypeSweepingCenterRange) {
+  SceneCameraMotionAnalyzerOptions options;
+  options.set_sweep_entire_frame(false);
+  TestableSceneCameraMotionAnalyzer analyzer(options);
+  auto scene_summary = GetDefaultSceneKeyFrameCropSummary();
+  scene_summary.set_frame_success_rate(
+      options.minimum_success_rate_for_sweeping() / 2.0f);
+  scene_summary.set_crop_window_width(kTargetWidth * 1.5f);
+  scene_summary.set_crop_window_height(kTargetHeight * 1.5f);
+  const double time_span = options.minimum_scene_span_sec_for_sweeping() * 2.0;
+  SceneCameraMotion camera_motion;
+
+  MP_EXPECT_OK(analyzer.DecideCameraMotionType(GetDefaultKeyFrameCropOptions(),
+                                               time_span, &scene_summary,
+                                               &camera_motion));
+
+  EXPECT_TRUE(camera_motion.has_sweeping_motion());
+  const auto& sweeping_motion = camera_motion.sweeping_motion();
+  EXPECT_FLOAT_EQ(sweeping_motion.sweep_start_center_x(),
+                  scene_summary.key_frame_center_min_x());
+  EXPECT_FLOAT_EQ(sweeping_motion.sweep_start_center_y(),
+                  scene_summary.key_frame_center_min_y());
+  EXPECT_FLOAT_EQ(sweeping_motion.sweep_end_center_x(),
+                  scene_summary.key_frame_center_max_x());
+  EXPECT_FLOAT_EQ(sweeping_motion.sweep_end_center_y(),
+                  scene_summary.key_frame_center_max_y());
+}
+
+// Checks that DecideCameraMotionType properly handles the case where motion is
+// small and there are no required focus regions.
+TEST(SceneCameraMotionAnalyzerTest,
+     DecideCameraMotionTypeSmallMotionNoRequiredFocusRegion) {
+  SceneCameraMotionAnalyzerOptions options;
+  options.set_motion_stabilization_threshold_percent(0.1);
+  options.set_snap_center_max_distance_percent(0.0);
+  TestableSceneCameraMotionAnalyzer analyzer(options);
+  auto scene_summary = GetSceneKeyFrameCropSummaryWithSmallMotion(options);
+  scene_summary.set_has_required_salient_region(false);
+  const int crop_region_center_x = 50;
+  SceneCameraMotion camera_motion;
+
+  MP_EXPECT_OK(analyzer.DecideCameraMotionType(GetDefaultKeyFrameCropOptions(),
+                                               kSceneTimeSpanSec,
+                                               &scene_summary, &camera_motion));
+  EXPECT_TRUE(camera_motion.has_steady_motion());
+  EXPECT_EQ(camera_motion.steady_motion().steady_look_at_center_x(),
+            crop_region_center_x);
+  EXPECT_EQ(scene_summary.crop_window_width(), kTargetWidth);
+  EXPECT_EQ(scene_summary.crop_window_height(), kTargetHeight);
+}
+
+// Checks that DecideCameraMotionType properly handles the case where motion is
+// small and there are required focus regions that fit in target size.
+TEST(SceneCameraMotionAnalyzerTest,
+     DecideCameraMotionTypeSmallMotionRequiredFocusRegionInTargetSize) {
+  SceneCameraMotionAnalyzerOptions options;
+  options.set_motion_stabilization_threshold_percent(0.1);
+  options.set_snap_center_max_distance_percent(0.0);
+  TestableSceneCameraMotionAnalyzer analyzer(options);
+  auto scene_summary = GetSceneKeyFrameCropSummaryWithSmallMotion(options);
+  scene_summary.set_has_required_salient_region(true);
+  *scene_summary.mutable_key_frame_required_crop_region_union() =
+      MakeRect(40, 0, 40, 10);
+  const int required_region_center_x = 60;
+  SceneCameraMotion camera_motion;
+
+  MP_EXPECT_OK(analyzer.DecideCameraMotionType(GetDefaultKeyFrameCropOptions(),
+                                               kSceneTimeSpanSec,
+                                               &scene_summary, &camera_motion));
+  EXPECT_TRUE(camera_motion.has_steady_motion());
+  EXPECT_EQ(camera_motion.steady_motion().steady_look_at_center_x(),
+            required_region_center_x);
+  EXPECT_EQ(scene_summary.crop_window_width(), 50);
+  EXPECT_EQ(scene_summary.crop_window_height(), kTargetHeight);
+}
+
+// Checks that DecideCameraMotionType properly handles the case where motion is
+// small and there are required focus regions that exceed target size.
+TEST(SceneCameraMotionAnalyzerTest,
+     DecideCameraMotionTypeSmallMotionRequiredFocusRegionExceedingTargetSize) {
+  SceneCameraMotionAnalyzerOptions options;
+  options.set_motion_stabilization_threshold_percent(0.1);
+  options.set_snap_center_max_distance_percent(0.0);
+  TestableSceneCameraMotionAnalyzer analyzer(options);
+  auto scene_summary = GetSceneKeyFrameCropSummaryWithSmallMotion(options);
+  scene_summary.set_has_required_salient_region(true);
+  *scene_summary.mutable_key_frame_required_crop_region_union() =
+      MakeRect(20, 0, 70, 10);
+  const int required_region_center_x = 55;
+  SceneCameraMotion camera_motion;
+
+  MP_EXPECT_OK(analyzer.DecideCameraMotionType(GetDefaultKeyFrameCropOptions(),
+                                               kSceneTimeSpanSec,
+                                               &scene_summary, &camera_motion));
+  EXPECT_TRUE(camera_motion.has_steady_motion());
+  EXPECT_EQ(camera_motion.steady_motion().steady_look_at_center_x(),
+            required_region_center_x);
+  EXPECT_EQ(scene_summary.crop_window_width(), 70);
+  EXPECT_EQ(scene_summary.crop_window_height(), kTargetHeight);
+}
+
+// Checks that DecideCameraMotionType properly handles the case where motion is
+// small and the middle of the key frame crop center range is close to the scene
+// frame center.
+TEST(SceneCameraMotionAnalyzerTest,
+     DecideCameraMotionTypeSmallMotionCloseToCenter) {
+  SceneCameraMotionAnalyzerOptions options;
+  options.set_motion_stabilization_threshold_percent(0.1);
+  options.set_snap_center_max_distance_percent(0.1);
+  TestableSceneCameraMotionAnalyzer analyzer(options);
+  KeyFrameCropOptions crop_options = GetDefaultKeyFrameCropOptions();
+  const float frame_center_x = kSceneFrameWidth / 2.0f;
+  auto scene_summary = GetSceneKeyFrameCropSummaryWithSmallMotion(options);
+  scene_summary.set_key_frame_center_min_x(frame_center_x - 2);
+  scene_summary.set_key_frame_center_max_x(frame_center_x);
+  SceneCameraMotion camera_motion;
+
+  MP_EXPECT_OK(analyzer.DecideCameraMotionType(crop_options, kSceneTimeSpanSec,
+                                               &scene_summary, &camera_motion));
+  EXPECT_TRUE(camera_motion.has_steady_motion());
+  EXPECT_FLOAT_EQ(camera_motion.steady_motion().steady_look_at_center_x(),
+                  frame_center_x);
+}
+
+// Checks that DecideCameraMotionType properly handles the case where motion is
+// not small, and sets the camera motion type to tracking.
+TEST(SceneCameraMotionAnalyzerTest, DecideCameraMotionTypeTracking) {
+  SceneCameraMotionAnalyzerOptions options;
+  TestableSceneCameraMotionAnalyzer analyzer(options);
+  auto scene_summary = GetDefaultSceneKeyFrameCropSummary();
+  scene_summary.set_horizontal_motion_amount(
+      options.motion_stabilization_threshold_percent() * 2.0);
+  SceneCameraMotion camera_motion;
+
+  MP_EXPECT_OK(analyzer.DecideCameraMotionType(GetDefaultKeyFrameCropOptions(),
+                                               kSceneTimeSpanSec,
+                                               &scene_summary, &camera_motion));
+  EXPECT_TRUE(camera_motion.has_tracking_motion());
+}
+
+// Checks that PopulateFocusPointFrames checks output pointer is not null.
+TEST(SceneCameraMotionAnalyzerTest,
+     PopulateFocusPointFramesChecksOutputNotNull) {
+  SceneCameraMotionAnalyzerOptions options;
+  TestableSceneCameraMotionAnalyzer analyzer(options);
+  SceneCameraMotion camera_motion;
+  const auto status = analyzer.PopulateFocusPointFrames(
+      GetDefaultSceneKeyFrameCropSummary(), camera_motion,
+      GetDefaultSceneFrameTimestamps(), nullptr);
+  EXPECT_FALSE(status.ok());
+  EXPECT_THAT(status.ToString(),
+              HasSubstr("Output vector of FocusPointFrame is null."));
+}
+
+// Checks that PopulateFocusPointFrames checks scene frames size.
+TEST(SceneCameraMotionAnalyzerTest,
+     PopulateFocusPointFramesChecksSceneFramesSize) {
+  SceneCameraMotionAnalyzerOptions options;
+  TestableSceneCameraMotionAnalyzer analyzer(options);
+  SceneCameraMotion camera_motion;
+  std::vector<int64> scene_frame_timestamps(0);
+  std::vector<FocusPointFrame> focus_point_frames;
+
+  const auto status = analyzer.PopulateFocusPointFrames(
+      GetDefaultSceneKeyFrameCropSummary(), camera_motion,
+      scene_frame_timestamps, &focus_point_frames);
+  EXPECT_FALSE(status.ok());
+  EXPECT_THAT(status.ToString(), HasSubstr("No scene frames."));
+}
+
+// Checks that PopulateFocusPointFrames handles the case of no key frames.
+TEST(SceneCameraMotionAnalyzerTest,
+     PopulateFocusPointFramesHandlesNoKeyFrames) {
+  SceneCameraMotionAnalyzerOptions options;
+  TestableSceneCameraMotionAnalyzer analyzer(options);
+  SceneCameraMotion camera_motion;
+  camera_motion.mutable_steady_motion();
+  auto scene_summary = GetDefaultSceneKeyFrameCropSummary();
+  scene_summary.set_num_key_frames(0);
+  scene_summary.clear_key_frame_compact_infos();
+  scene_summary.set_has_salient_region(false);
+  std::vector<FocusPointFrame> focus_point_frames;
+  MP_EXPECT_OK(analyzer.PopulateFocusPointFrames(
+      scene_summary, camera_motion, GetDefaultSceneFrameTimestamps(),
+      &focus_point_frames));
+}
+
+// Checks that PopulateFocusPointFrames checks KeyFrameCompactInfos has the
+// right size.
+TEST(SceneCameraMotionAnalyzerTest,
+     PopulateFocusPointFramesChecksKeyFrameCompactInfosSize) {
+  SceneCameraMotionAnalyzerOptions options;
+  TestableSceneCameraMotionAnalyzer analyzer(options);
+  SceneCameraMotion camera_motion;
+  auto scene_summary = GetDefaultSceneKeyFrameCropSummary();
+  scene_summary.set_num_key_frames(2 * kNumKeyFrames);
+  std::vector<FocusPointFrame> focus_point_frames;
+
+  const auto status = analyzer.PopulateFocusPointFrames(
+      scene_summary, camera_motion, GetDefaultSceneFrameTimestamps(),
+      &focus_point_frames);
+  EXPECT_FALSE(status.ok());
+  EXPECT_THAT(status.ToString(),
+              HasSubstr("Key frame compact infos has wrong size"));
+}
+
+// Checks that PopulateFocusPointFrames checks SceneKeyFrameCropSummary has
+// valid scene frame size.
+TEST(SceneCameraMotionAnalyzerTest,
+     PopulateFocusPointFramesChecksSceneFrameSize) {
+  SceneCameraMotionAnalyzerOptions options;
+  TestableSceneCameraMotionAnalyzer analyzer(options);
+  SceneCameraMotion camera_motion;
+  auto scene_summary = GetDefaultSceneKeyFrameCropSummary();
+  scene_summary.set_scene_frame_height(0);
+  std::vector<FocusPointFrame> focus_point_frames;
+
+  const auto status = analyzer.PopulateFocusPointFrames(
+      scene_summary, camera_motion, GetDefaultSceneFrameTimestamps(),
+      &focus_point_frames);
+  EXPECT_FALSE(status.ok());
+  EXPECT_THAT(status.ToString(), HasSubstr("Non-positive frame height."));
+}
+
+// Checks that PopulateFocusPointFrames checks camera motion type is valid.
+TEST(SceneCameraMotionAnalyzerTest,
+     PopulateFocusPointFramesChecksCameraMotionType) {
+  SceneCameraMotionAnalyzerOptions options;
+  TestableSceneCameraMotionAnalyzer analyzer(options);
+  SceneCameraMotion camera_motion;
+  camera_motion.clear_motion_type();
+  std::vector<FocusPointFrame> focus_point_frames;
+
+  const auto status = analyzer.PopulateFocusPointFrames(
+      GetDefaultSceneKeyFrameCropSummary(), camera_motion,
+      GetDefaultSceneFrameTimestamps(), &focus_point_frames);
+  EXPECT_FALSE(status.ok());
+  EXPECT_THAT(status.ToString(), HasSubstr("Unknown motion type."));
+}
+
+// Checks that PopulateFocusPointFrames properly sets FocusPointFrames when
+// camera motion type is steady.
+TEST(SceneCameraMotionAnalyzerTest, PopulateFocusPointFramesSteady) {
+  SceneCameraMotionAnalyzerOptions options;
+  TestableSceneCameraMotionAnalyzer analyzer(options);
+  SceneCameraMotion camera_motion;
+  auto* steady_motion = camera_motion.mutable_steady_motion();
+  steady_motion->set_steady_look_at_center_x(40.5);
+  steady_motion->set_steady_look_at_center_y(25);
+  std::vector<FocusPointFrame> focus_point_frames;
+
+  MP_EXPECT_OK(analyzer.PopulateFocusPointFrames(
+      GetDefaultSceneKeyFrameCropSummary(), camera_motion,
+      GetDefaultSceneFrameTimestamps(), &focus_point_frames));
+
+  EXPECT_EQ(kNumSceneFrames, focus_point_frames.size());
+  for (int i = 0; i < kNumSceneFrames; ++i) {
+    // FocusPointFrameType is CENTER.
+    EXPECT_EQ(focus_point_frames[i].point_size(), 1);
+    const auto& point = focus_point_frames[i].point(0);
+    EXPECT_FLOAT_EQ(
+        point.norm_point_x(),
+        steady_motion->steady_look_at_center_x() / kSceneFrameWidth);
+    EXPECT_FLOAT_EQ(
+        point.norm_point_y(),
+        steady_motion->steady_look_at_center_y() / kSceneFrameHeight);
+    EXPECT_FLOAT_EQ(point.weight(), options.maximum_salient_point_weight());
+  }
+}
+
+// Checks that PopulateFocusPointFrames properly sets FocusPointFrames when
+// FocusPointFrameType is TOPMOST_AND_BOTTOMMOST.
+TEST(SceneCameraMotionAnalyzerTest, PopulateFocusPointFramesTopAndBottom) {
+  SceneCameraMotionAnalyzerOptions options;
+  TestableSceneCameraMotionAnalyzer analyzer(options);
+  SceneCameraMotion camera_motion;
+  auto* steady_motion = camera_motion.mutable_steady_motion();
+  steady_motion->set_steady_look_at_center_x(40.5);
+  steady_motion->set_steady_look_at_center_y(25);
+  // Forces FocusPointFrameType to be TOPMOST_AND_BOTTOMOST.
+  auto scene_summary = GetDefaultSceneKeyFrameCropSummary();
+  scene_summary.set_crop_window_height(kSceneFrameHeight);
+  std::vector<FocusPointFrame> focus_point_frames;
+
+  MP_EXPECT_OK(analyzer.PopulateFocusPointFrames(
+      scene_summary, camera_motion, GetDefaultSceneFrameTimestamps(),
+      &focus_point_frames));
+
+  EXPECT_EQ(kNumSceneFrames, focus_point_frames.size());
+  for (int i = 0; i < kNumSceneFrames; ++i) {
+    EXPECT_EQ(focus_point_frames[i].point_size(), 2);
+    const auto& point1 = focus_point_frames[i].point(0);
+    const auto& point2 = focus_point_frames[i].point(1);
+    EXPECT_FLOAT_EQ(
+        point1.norm_point_x(),
+        steady_motion->steady_look_at_center_x() / kSceneFrameWidth);
+    EXPECT_FLOAT_EQ(point1.norm_point_y(), 0.0f);
+    EXPECT_FLOAT_EQ(
+        point2.norm_point_x(),
+        steady_motion->steady_look_at_center_x() / kSceneFrameWidth);
+    EXPECT_FLOAT_EQ(point2.norm_point_y(), 1.0f);
+    EXPECT_FLOAT_EQ(point1.weight(), options.maximum_salient_point_weight());
+    EXPECT_FLOAT_EQ(point2.weight(), options.maximum_salient_point_weight());
+  }
+}
+
+// Checks that PopulateFocusPointFrames properly sets FocusPointFrames when
+// FocusPointFrameType is LEFTMOST_AND_RIGHTMOST.
+TEST(SceneCameraMotionAnalyzerTest, PopulateFocusPointFramesLeftAndRight) {
+  SceneCameraMotionAnalyzerOptions options;
+  TestableSceneCameraMotionAnalyzer analyzer(options);
+  SceneCameraMotion camera_motion;
+  auto* steady_motion = camera_motion.mutable_steady_motion();
+  steady_motion->set_steady_look_at_center_x(40.5);
+  steady_motion->set_steady_look_at_center_y(25);
+  // Forces FocusPointFrameType to be LEFTMOST_AND_RIGHTMOST.
+  auto scene_summary = GetDefaultSceneKeyFrameCropSummary();
+  scene_summary.set_crop_window_width(kSceneFrameWidth);
+  std::vector<FocusPointFrame> focus_point_frames;
+
+  MP_EXPECT_OK(analyzer.PopulateFocusPointFrames(
+      scene_summary, camera_motion, GetDefaultSceneFrameTimestamps(),
+      &focus_point_frames));
+
+  EXPECT_EQ(kNumSceneFrames, focus_point_frames.size());
+  for (int i = 0; i < kNumSceneFrames; ++i) {
+    EXPECT_EQ(focus_point_frames[i].point_size(), 2);
+    const auto& point1 = focus_point_frames[i].point(0);
+    const auto& point2 = focus_point_frames[i].point(1);
+    EXPECT_FLOAT_EQ(point1.norm_point_x(), 0.0f);
+    EXPECT_FLOAT_EQ(
+        point1.norm_point_y(),
+        steady_motion->steady_look_at_center_y() / kSceneFrameHeight);
+    EXPECT_FLOAT_EQ(point2.norm_point_x(), 1.0f);
+    EXPECT_FLOAT_EQ(
+        point1.norm_point_y(),
+        steady_motion->steady_look_at_center_y() / kSceneFrameHeight);
+    EXPECT_FLOAT_EQ(point1.weight(), options.maximum_salient_point_weight());
+    EXPECT_FLOAT_EQ(point2.weight(), options.maximum_salient_point_weight());
+  }
+}
+
+// Checks that PopulateFocusPointFrames properly sets FocusPointFrames when
+// camera motion type is sweeping.
+TEST(SceneCameraMotionAnalyzerTest, PopulateFocusPointFramesSweeping) {
+  SceneCameraMotionAnalyzerOptions options;
+  TestableSceneCameraMotionAnalyzer analyzer(options);
+  SceneCameraMotion camera_motion;
+  auto* sweeping_motion = camera_motion.mutable_sweeping_motion();
+  sweeping_motion->set_sweep_start_center_x(5);
+  sweeping_motion->set_sweep_start_center_y(50);
+  sweeping_motion->set_sweep_end_center_x(95);
+  sweeping_motion->set_sweep_end_center_y(50);
+  const int num_frames = 10;
+  const std::vector<float> positions_x = {5,  15, 25, 35, 45,
+                                          55, 65, 75, 85, 95};
+  const std::vector<float> positions_y = {50, 50, 50, 50, 50,
+                                          50, 50, 50, 50, 50};
+  std::vector<int64> scene_frame_timestamps(num_frames);
+  std::iota(scene_frame_timestamps.begin(), scene_frame_timestamps.end(), 0);
+  std::vector<FocusPointFrame> focus_point_frames;
+
+  MP_EXPECT_OK(analyzer.PopulateFocusPointFrames(
+      GetDefaultSceneKeyFrameCropSummary(), camera_motion,
+      scene_frame_timestamps, &focus_point_frames));
+
+  EXPECT_EQ(num_frames, focus_point_frames.size());
+  for (int i = 0; i < num_frames; ++i) {
+    EXPECT_EQ(focus_point_frames[i].point_size(), 1);
+    const auto& point = focus_point_frames[i].point(0);
+    EXPECT_FLOAT_EQ(positions_x[i] / kSceneFrameWidth, point.norm_point_x());
+    EXPECT_FLOAT_EQ(positions_y[i] / kSceneFrameHeight, point.norm_point_y());
+  }
+}
+
+// Checks that PopulateFocusPointFrames checks tracking handles the case when
+// maximum score is 0.
+TEST(SceneCameraMotionAnalyzerTest,
+     PopulateFocusPointFramesTrackingHandlesZeroScore) {
+  SceneCameraMotionAnalyzerOptions options;
+  TestableSceneCameraMotionAnalyzer analyzer(options);
+  SceneCameraMotion camera_motion;
+  camera_motion.mutable_tracking_motion();
+  auto scene_summary = GetDefaultSceneKeyFrameCropSummary();
+  scene_summary.set_key_frame_max_score(0.0);
+  for (int i = 0; i < kNumKeyFrames; ++i) {
+    scene_summary.mutable_key_frame_compact_infos(i)->set_score(0.0);
+  }
+  std::vector<FocusPointFrame> focus_point_frames;
+  MP_EXPECT_OK(analyzer.PopulateFocusPointFrames(
+      scene_summary, camera_motion, GetDefaultSceneFrameTimestamps(),
+      &focus_point_frames));
+}
+
+// Checks that PopulateFocusPointFrames skips empty key frames when camera
+// motion type is tracking.
+TEST(SceneCameraMotionAnalyzerTest,
+     PopulateFocusPointFramesTrackingSkipsEmptyKeyFrames) {
+  SceneCameraMotionAnalyzerOptions options;
+  TestableSceneCameraMotionAnalyzer analyzer(options);
+  SceneCameraMotion camera_motion;
+  camera_motion.mutable_tracking_motion();
+  SceneKeyFrameCropSummary scene_summary;
+  scene_summary.set_scene_frame_width(kSceneFrameWidth);
+  scene_summary.set_scene_frame_height(kSceneFrameHeight);
+  scene_summary.set_num_key_frames(2);
+
+  // Sets first key frame to be empty and second frame to be normal.
+  const float center_x = 25.0f, center_y = 25.0f;
+  auto* first_frame_compact_info = scene_summary.add_key_frame_compact_infos();
+  first_frame_compact_info->set_center_x(-1.0);
+  auto* second_frame_compact_info = scene_summary.add_key_frame_compact_infos();
+  second_frame_compact_info->set_center_x(center_x);
+  second_frame_compact_info->set_center_y(center_y);
+  second_frame_compact_info->set_score(1.0);
+  scene_summary.set_key_frame_center_min_x(center_x);
+  scene_summary.set_key_frame_center_max_x(center_x);
+  scene_summary.set_key_frame_center_min_y(center_y);
+  scene_summary.set_key_frame_center_max_y(center_y);
+  scene_summary.set_key_frame_min_score(1.0);
+  scene_summary.set_key_frame_max_score(1.0);
+
+  // Aligns timestamps of scene frames with key frames.
+  scene_summary.mutable_key_frame_compact_infos(0)->set_timestamp_ms(10);
+  scene_summary.mutable_key_frame_compact_infos(1)->set_timestamp_ms(20);
+  std::vector<int64> scene_frame_timestamps = {10, 20};
+
+  std::vector<FocusPointFrame> focus_point_frames;
+  MP_EXPECT_OK(analyzer.PopulateFocusPointFrames(scene_summary, camera_motion,
+                                                 scene_frame_timestamps,
+                                                 &focus_point_frames));
+
+  // Both scene frames should have focus point frames based on the second key
+  // frame since the first one is empty and not used.
+  for (int i = 0; i < 2; ++i) {
+    EXPECT_EQ(focus_point_frames[i].point_size(), 1);
+    const auto& point = focus_point_frames[i].point(0);
+    EXPECT_FLOAT_EQ(point.norm_point_x(), center_x / kSceneFrameWidth);
+    EXPECT_FLOAT_EQ(point.norm_point_y(), center_y / kSceneFrameHeight);
+    EXPECT_FLOAT_EQ(point.weight(), options.maximum_salient_point_weight());
+  }
+}
+
+// Checks that PopulateFocusPointFrames properly sets FocusPointFrames when
+// camera motion type is tracking, piecewise-linearly interpolating key frame
+// centers and scores, and scaling scores so that maximum weight is equal to
+// maximum_salient_point_weight.
+TEST(SceneCameraMotionAnalyzerTest,
+     PopulateFocusPointFramesTrackingTracksKeyFrames) {
+  SceneCameraMotionAnalyzerOptions options;
+  TestableSceneCameraMotionAnalyzer analyzer(options);
+  SceneCameraMotion camera_motion;
+  camera_motion.mutable_tracking_motion();
+  auto scene_summary = GetDefaultSceneKeyFrameCropSummary();
+  const std::vector<float> centers_x = {14.0, 5.0, 40.0, 70.0, 30.0};
+  const std::vector<float> centers_y = {60.0, 50.0, 80.0, 0.0, 20.0};
+  const std::vector<float> scores = {0.1, 1.0, 2.0, 0.6, 0.9};
+  scene_summary.set_key_frame_min_score(0.1);
+  scene_summary.set_key_frame_max_score(2.0);
+  for (int i = 0; i < kNumKeyFrames; ++i) {
+    auto* compact_info = scene_summary.mutable_key_frame_compact_infos(i);
+    compact_info->set_center_x(centers_x[i]);
+    compact_info->set_center_y(centers_y[i]);
+    compact_info->set_score(scores[i]);
+  }
+
+  // Get reference scene frame results from csv file.
+  const std::string scene_frame_results_file_path =
+      mediapipe::file::JoinPath("./", kCameraTrackingSceneFrameResultsFile);
+  std::string csv_file_content;
+  MP_ASSERT_OK(mediapipe::file::GetContents(scene_frame_results_file_path,
+                                            &csv_file_content));
+  std::vector<std::string> lines = absl::StrSplit(csv_file_content, '\n');
+  std::vector<std::string> records;
+  for (const auto& line : lines) {
+    std::vector<std::string> r = absl::StrSplit(line, ',');
+    records.insert(records.end(), r.begin(), r.end());
+  }
+  CHECK_EQ(records.size(), kNumSceneFrames * 3 + 1);
+
+  std::vector<FocusPointFrame> focus_point_frames;
+  MP_EXPECT_OK(analyzer.PopulateFocusPointFrames(
+      scene_summary, camera_motion, GetDefaultSceneFrameTimestamps(),
+      &focus_point_frames));
+
+  float max_weight = 0.0;
+  const float tolerance = 1e-4;
+  for (int i = 0; i < kNumSceneFrames; ++i) {
+    EXPECT_EQ(focus_point_frames[i].point_size(), 1);
+    const auto& point = focus_point_frames[i].point(0);
+    const float expected_x = std::stof(records[i * 3]);
+    const float expected_y = std::stof(records[i * 3 + 1]);
+    const float expected_weight = std::stof(records[i * 3 + 2]);
+    EXPECT_LE(std::fabs(point.norm_point_x() - expected_x), tolerance);
+    EXPECT_LE(std::fabs(point.norm_point_y() - expected_y), tolerance);
+    EXPECT_LE(std::fabs(point.weight() - expected_weight), tolerance);
+    max_weight = std::max(max_weight, point.weight());
+  }
+  EXPECT_LE(std::fabs(max_weight - options.maximum_salient_point_weight()),
+            tolerance);
+}
+
+// Checks that AnalyzeSceneAndPopulateFocusPointFrames analyzes scene and
+// populates focus point frames.
+TEST(SceneCameraMotionAnalyzerTest, AnalyzeSceneAndPopulateFocusPointFrames) {
+  SceneCameraMotionAnalyzerOptions options;
+  SceneCameraMotionAnalyzer analyzer(options);
+  SceneKeyFrameCropSummary scene_summary;
+  std::vector<FocusPointFrame> focus_point_frames;
+
+  MP_EXPECT_OK(analyzer.AnalyzeSceneAndPopulateFocusPointFrames(
+      GetDefaultKeyFrameInfos(), GetDefaultKeyFrameCropOptions(),
+      GetDefaultKeyFrameCropResults(), kSceneFrameWidth, kSceneFrameHeight,
+      GetDefaultSceneFrameTimestamps(), &scene_summary, &focus_point_frames));
+  EXPECT_EQ(scene_summary.num_key_frames(), kNumKeyFrames);
+  EXPECT_EQ(focus_point_frames.size(), kNumSceneFrames);
+}
+
+// Checks that AnalyzeSceneAndPopulateFocusPointFrames optionally returns
+// scene camera motion.
+TEST(SceneCameraMotionAnalyzerTest,
+     AnalyzeSceneAndPopulateFocusPointFramesReturnsSceneCameraMotion) {
+  SceneCameraMotionAnalyzerOptions options;
+  SceneCameraMotionAnalyzer analyzer(options);
+  SceneKeyFrameCropSummary scene_summary;
+  std::vector<FocusPointFrame> focus_point_frames;
+  SceneCameraMotion scene_camera_motion;
+
+  MP_EXPECT_OK(analyzer.AnalyzeSceneAndPopulateFocusPointFrames(
+      GetDefaultKeyFrameInfos(), GetDefaultKeyFrameCropOptions(),
+      GetDefaultKeyFrameCropResults(), kSceneFrameWidth, kSceneFrameHeight,
+      GetDefaultSceneFrameTimestamps(), &scene_summary, &focus_point_frames,
+      &scene_camera_motion));
+  EXPECT_TRUE(scene_camera_motion.has_steady_motion());
+}
+
+}  // namespace autoflip
+}  // namespace mediapipe

mediapipe/examples/desktop/autoflip/quality/scene_cropper.cc

@@ -0,0 +1,84 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/examples/desktop/autoflip/quality/scene_cropper.h"
+
+#include "absl/memory/memory.h"
+#include "mediapipe/examples/desktop/autoflip/quality/polynomial_regression_path_solver.h"
+#include "mediapipe/examples/desktop/autoflip/quality/utils.h"
+#include "mediapipe/framework/port/opencv_core_inc.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+namespace autoflip {
+
+::mediapipe::Status SceneCropper::CropFrames(
+    const SceneKeyFrameCropSummary& scene_summary,
+    const std::vector<cv::Mat>& scene_frames,
+    const std::vector<FocusPointFrame>& focus_point_frames,
+    const std::vector<FocusPointFrame>& prior_focus_point_frames,
+    std::vector<cv::Mat>* cropped_frames) const {
+  RET_CHECK_NE(cropped_frames, nullptr) << "Output cropped frames is null.";
+
+  const int num_scene_frames = scene_frames.size();
+  RET_CHECK_GT(num_scene_frames, 0) << "No scene frames.";
+  RET_CHECK_EQ(focus_point_frames.size(), num_scene_frames)
+      << "Wrong size of FocusPointFrames.";
+
+  const int frame_width = scene_summary.scene_frame_width();
+  const int frame_height = scene_summary.scene_frame_height();
+  const int crop_width = scene_summary.crop_window_width();
+  const int crop_height = scene_summary.crop_window_height();
+  RET_CHECK_GT(crop_width, 0) << "Crop width is non-positive.";
+  RET_CHECK_GT(crop_height, 0) << "Crop height is non-positive.";
+  RET_CHECK_LE(crop_width, frame_width) << "Crop width exceeds frame width.";
+  RET_CHECK_LE(crop_height, frame_height)
+      << "Crop height exceeds frame height.";
+
+  // Computes transforms.
+  std::vector<cv::Mat> all_xforms;
+
+  PolynomialRegressionPathSolver solver;
+  RET_CHECK_OK(solver.ComputeCameraPath(
+      focus_point_frames, prior_focus_point_frames, frame_width, frame_height,
+      crop_width, crop_height, &all_xforms));
+
+  const int num_prior = prior_focus_point_frames.size();
+  std::vector<cv::Mat> scene_frame_xforms(all_xforms.begin() + num_prior,
+                                          all_xforms.end());
+
+  // Convert the matrix from center-aligned to upper-left aligned.
+  for (cv::Mat& xform : scene_frame_xforms) {
+    cv::Mat affine_opencv = cv::Mat::eye(2, 3, CV_32FC1);
+    affine_opencv.at<float>(0, 2) =
+        -(xform.at<float>(0, 2) + frame_width / 2 - crop_width / 2);
+    affine_opencv.at<float>(1, 2) =
+        -(xform.at<float>(1, 2) + frame_height / 2 - crop_height / 2);
+    xform = affine_opencv;
+  }
+
+  // Prepares cropped frames.
+  cropped_frames->resize(num_scene_frames);
+  for (int i = 0; i < num_scene_frames; ++i) {
+    (*cropped_frames)[i] =
+        cv::Mat::zeros(crop_height, crop_width, scene_frames[i].type());
+  }
+
+  return AffineRetarget(cv::Size(crop_width, crop_height), scene_frames,
+                        scene_frame_xforms, cropped_frames);
+}
+
+}  // namespace autoflip
+}  // namespace mediapipe

mediapipe/examples/desktop/autoflip/quality/scene_cropper.h

@@ -0,0 +1,65 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_SCENE_CROPPER_H_
+#define MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_SCENE_CROPPER_H_
+
+#include <memory>
+#include <vector>
+
+#include "mediapipe/examples/desktop/autoflip/quality/cropping.pb.h"
+#include "mediapipe/examples/desktop/autoflip/quality/focus_point.pb.h"
+#include "mediapipe/framework/port/opencv_core_inc.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+namespace autoflip {
+
+// This class is a thin wrapper around the Retargeter class to crop a collection
+// of scene frames given SceneKeyFrameCropSummary and their FocusPointFrames.
+//
+// Upstream inputs:
+// - SceneKeyFrameCropSummary scene_summary.
+// - std::vector<FocusPointFrame> focus_point_frames.
+// - std::vector<FocusPointFrame> prior_focus_point_frames.
+// - std::vector<cv::Mat> scene_frames;
+//
+// Example usage:
+//   SceneCropperOptions scene_cropper_options;
+//   SceneCropper scene_cropper(scene_cropper_options);
+//   std::vector<cv::Mat> cropped_frames;
+//   CHECK_OK(scene_cropper.CropFrames(
+//       scene_summary, scene_frames, focus_point_frames,
+//       prior_focus_point_frames, &cropped_frames));
+class SceneCropper {
+ public:
+  SceneCropper() {}
+  ~SceneCropper() {}
+
+  // Crops scene frames given SceneKeyFrameCropSummary, FocusPointFrames, and
+  // any prior FocusPointFrames (to ensure smoothness when there was no actual
+  // scene change).
+  ::mediapipe::Status CropFrames(
+      const SceneKeyFrameCropSummary& scene_summary,
+      const std::vector<cv::Mat>& scene_frames,
+      const std::vector<FocusPointFrame>& focus_point_frames,
+      const std::vector<FocusPointFrame>& prior_focus_point_frames,
+      std::vector<cv::Mat>* cropped_frames) const;
+};
+
+}  // namespace autoflip
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_SCENE_CROPPER_H_

mediapipe/examples/desktop/autoflip/quality/scene_cropper_test.cc

@@ -0,0 +1,164 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/examples/desktop/autoflip/quality/scene_cropper.h"
+
+#include "mediapipe/examples/desktop/autoflip/quality/focus_point.pb.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/opencv_core_inc.h"
+#include "mediapipe/framework/port/status_matchers.h"
+
+namespace mediapipe {
+namespace autoflip {
+
+using testing::HasSubstr;
+
+const int kCropWidth = 90;
+const int kCropHeight = 160;
+
+const int kSceneWidth = 320;
+const int kSceneHeight = 180;
+
+const int kNumSceneFrames = 30;
+
+// Returns default values for SceneKeyFrameCropSummary. Sets scene size and crop
+// window size from default values.
+SceneKeyFrameCropSummary GetDefaultSceneKeyFrameCropSummary() {
+  SceneKeyFrameCropSummary scene_summary;
+  scene_summary.set_scene_frame_width(kSceneWidth);
+  scene_summary.set_scene_frame_height(kSceneHeight);
+  scene_summary.set_crop_window_width(kCropWidth);
+  scene_summary.set_crop_window_height(kCropHeight);
+  return scene_summary;
+}
+
+// Returns default values for scene frames of size kNumSceneFrames. Stes each
+// frame to be solid red color at default scene size.
+std::vector<cv::Mat> GetDefaultSceneFrames() {
+  std::vector<cv::Mat> scene_frames(kNumSceneFrames);
+  for (int i = 0; i < kNumSceneFrames; ++i) {
+    scene_frames[i] = cv::Mat(kSceneHeight, kSceneWidth, CV_8UC3);
+    scene_frames[i] = cv::Scalar(255, 0, 0);
+  }
+  return scene_frames;
+}
+
+// Makes a vector of FocusPointFrames given size. Stes each FocusPointFrame
+// to have one FocusPoint at the center of the frame.
+std::vector<FocusPointFrame> GetFocusPointFrames(const int num_frames) {
+  std::vector<FocusPointFrame> focus_point_frames(num_frames);
+  for (int i = 0; i < num_frames; ++i) {
+    auto* point = focus_point_frames[i].add_point();
+    point->set_norm_point_x(0.5);
+    point->set_norm_point_y(0.5);
+  }
+  return focus_point_frames;
+}
+// Returns default values for FocusPointFrames of size kNumSceneFrames.
+std::vector<FocusPointFrame> GetDefaultFocusPointFrames() {
+  return GetFocusPointFrames(kNumSceneFrames);
+}
+
+// Checks that CropFrames checks output pointer is not null.
+TEST(SceneCropperTest, CropFramesChecksOutputNotNull) {
+  SceneCropper scene_cropper;
+  const auto status = scene_cropper.CropFrames(
+      GetDefaultSceneKeyFrameCropSummary(), GetDefaultSceneFrames(),
+      GetDefaultFocusPointFrames(), GetFocusPointFrames(0), nullptr);
+  EXPECT_FALSE(status.ok());
+  EXPECT_THAT(status.ToString(), HasSubstr("Output cropped frames is null."));
+}
+
+// Checks that CropFrames checks that scene frames size is positive.
+TEST(SceneCropperTest, CropFramesChecksSceneFramesSize) {
+  SceneCropper scene_cropper;
+  std::vector<cv::Mat> scene_frames(0);
+  std::vector<cv::Mat> cropped_frames;
+  const auto status = scene_cropper.CropFrames(
+      GetDefaultSceneKeyFrameCropSummary(), scene_frames,
+      GetDefaultFocusPointFrames(), GetFocusPointFrames(0), &cropped_frames);
+  EXPECT_FALSE(status.ok());
+  EXPECT_THAT(status.ToString(), HasSubstr("No scene frames."));
+}
+
+// Checks that CropFrames checks that FocusPointFrames has the right size.
+TEST(SceneCropperTest, CropFramesChecksFocusPointFramesSize) {
+  SceneCropper scene_cropper;
+  std::vector<cv::Mat> cropped_frames;
+  const auto status = scene_cropper.CropFrames(
+      GetDefaultSceneKeyFrameCropSummary(), GetDefaultSceneFrames(),
+      GetFocusPointFrames(kNumSceneFrames - 1), GetFocusPointFrames(0),
+      &cropped_frames);
+  EXPECT_FALSE(status.ok());
+  EXPECT_THAT(status.ToString(), HasSubstr("Wrong size of FocusPointFrames"));
+}
+
+// Checks that CropFrames checks crop size is positive.
+TEST(SceneCropperTest, CropFramesChecksCropSizePositive) {
+  auto scene_summary = GetDefaultSceneKeyFrameCropSummary();
+  scene_summary.set_crop_window_width(-1);
+  SceneCropper scene_cropper;
+  std::vector<cv::Mat> cropped_frames;
+  const auto status = scene_cropper.CropFrames(
+      scene_summary, GetDefaultSceneFrames(), GetDefaultFocusPointFrames(),
+      GetFocusPointFrames(0), &cropped_frames);
+  EXPECT_FALSE(status.ok());
+  EXPECT_THAT(status.ToString(), HasSubstr("Crop width is non-positive."));
+}
+
+// Checks that CropFrames checks that crop size does not exceed frame size.
+TEST(SceneCropperTest, InitializesRetargeterChecksCropSizeNotExceedFrameSize) {
+  auto scene_summary = GetDefaultSceneKeyFrameCropSummary();
+  scene_summary.set_crop_window_height(kSceneHeight + 1);
+  SceneCropper scene_cropper;
+  std::vector<cv::Mat> cropped_frames;
+  const auto status = scene_cropper.CropFrames(
+      scene_summary, GetDefaultSceneFrames(), GetDefaultFocusPointFrames(),
+      GetFocusPointFrames(0), &cropped_frames);
+  EXPECT_FALSE(status.ok());
+  EXPECT_THAT(status.ToString(),
+              HasSubstr("Crop height exceeds frame height."));
+}
+
+// Checks that CropFrames works when there are not any prior FocusPointFrames.
+TEST(SceneCropperTest, CropFramesWorksWithoutPriorFocusPointFrames) {
+  SceneCropper scene_cropper;
+  std::vector<cv::Mat> cropped_frames;
+  MP_ASSERT_OK(scene_cropper.CropFrames(
+      GetDefaultSceneKeyFrameCropSummary(), GetDefaultSceneFrames(),
+      GetDefaultFocusPointFrames(), GetFocusPointFrames(0), &cropped_frames));
+  ASSERT_EQ(cropped_frames.size(), kNumSceneFrames);
+  for (int i = 0; i < kNumSceneFrames; ++i) {
+    EXPECT_EQ(cropped_frames[i].rows, kCropHeight);
+    EXPECT_EQ(cropped_frames[i].cols, kCropWidth);
+  }
+}
+
+// Checks that CropFrames works when there are prior FocusPointFrames.
+TEST(SceneCropperTest, CropFramesWorksWithPriorFocusPointFrames) {
+  SceneCropper scene_cropper;
+  std::vector<cv::Mat> cropped_frames;
+  MP_EXPECT_OK(scene_cropper.CropFrames(
+      GetDefaultSceneKeyFrameCropSummary(), GetDefaultSceneFrames(),
+      GetDefaultFocusPointFrames(), GetFocusPointFrames(3), &cropped_frames));
+  EXPECT_EQ(cropped_frames.size(), kNumSceneFrames);
+  for (int i = 0; i < kNumSceneFrames; ++i) {
+    EXPECT_EQ(cropped_frames[i].rows, kCropHeight);
+    EXPECT_EQ(cropped_frames[i].cols, kCropWidth);
+  }
+}
+
+}  // namespace autoflip
+}  // namespace mediapipe

mediapipe/examples/desktop/autoflip/quality/scene_cropping_viz.cc

@@ -0,0 +1,186 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/examples/desktop/autoflip/quality/scene_cropping_viz.h"
+
+#include <memory>
+
+#include "absl/memory/memory.h"
+#include "mediapipe/examples/desktop/autoflip/autoflip_messages.pb.h"
+#include "mediapipe/examples/desktop/autoflip/quality/cropping.pb.h"
+#include "mediapipe/examples/desktop/autoflip/quality/focus_point.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/image_format.pb.h"
+#include "mediapipe/framework/formats/image_frame.h"
+#include "mediapipe/framework/formats/image_frame_opencv.h"
+#include "mediapipe/framework/port/opencv_core_inc.h"
+#include "mediapipe/framework/port/opencv_imgproc_inc.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+namespace autoflip {
+
+// Colors for focus signal sources.
+const cv::Scalar kCyan =
+    cv::Scalar(0.0, 255.0, 255.0);  // brain object detector
+const cv::Scalar kMagenta = cv::Scalar(255.0, 0.0, 255.0);        // motion
+const cv::Scalar kYellow = cv::Scalar(255.0, 255.0, 0.0);         // fg ocr
+const cv::Scalar kLightYellow = cv::Scalar(255.0, 250.0, 205.0);  // bg ocr
+const cv::Scalar kRed = cv::Scalar(255.0, 0.0, 0.0);              // logo
+const cv::Scalar kGreen = cv::Scalar(0.0, 255.0, 0.0);            // face
+const cv::Scalar kBlue =
+    cv::Scalar(0.0, 0.0, 255.0);  // creatism saliency model
+const cv::Scalar kOrange =
+    cv::Scalar(255.0, 165.0, 0.0);  // ica object detector
+const cv::Scalar kWhite = cv::Scalar(255.0, 255.0, 255.0);  // others
+
+::mediapipe::Status DrawDetectionsAndCropRegions(
+    const std::vector<cv::Mat>& scene_frames,
+    const std::vector<bool>& is_key_frames,
+    const std::vector<KeyFrameInfo>& key_frame_infos,
+    const std::vector<KeyFrameCropResult>& key_frame_crop_results,
+    const ImageFormat::Format image_format,
+    std::vector<std::unique_ptr<ImageFrame>>* viz_frames) {
+  RET_CHECK(viz_frames) << "Output viz frames is null.";
+  viz_frames->clear();
+  const int num_frames = scene_frames.size();
+
+  std::pair<cv::Point, cv::Point> crop_corners;
+  std::vector<std::pair<cv::Point, cv::Point>> region_corners;
+  std::vector<cv::Scalar> region_colors;
+  auto RectToCvPoints =
+      [](const Rect& rect) -> std::pair<cv::Point, cv::Point> {
+    return std::make_pair(
+        cv::Point(rect.x(), rect.y()),
+        cv::Point(rect.x() + rect.width(), rect.y() + rect.height()));
+  };
+
+  int key_frame_idx = 0;
+  for (int i = 0; i < num_frames; ++i) {
+    const auto& scene_frame = scene_frames[i];
+    auto viz_frame = absl::make_unique<ImageFrame>(
+        image_format, scene_frame.cols, scene_frame.rows);
+    cv::Mat viz_mat = formats::MatView(viz_frame.get());
+    scene_frame.copyTo(viz_mat);
+
+    if (is_key_frames[i]) {
+      const auto& bbox = key_frame_crop_results[key_frame_idx].region();
+      crop_corners = RectToCvPoints(bbox);
+      region_corners.clear();
+      region_colors.clear();
+      const auto& detections = key_frame_infos[key_frame_idx].detections();
+      for (int j = 0; j < detections.detections_size(); ++j) {
+        const auto& detection = detections.detections(j);
+        const auto corners = RectToCvPoints(detection.location());
+        region_corners.push_back(corners);
+        if (detection.signal_type().has_standard()) {
+          switch (detection.signal_type().standard()) {
+            case SignalType::FACE_FULL:
+            case SignalType::FACE_LANDMARK:
+            case SignalType::FACE_ALL_LANDMARKS:
+            case SignalType::FACE_CORE_LANDMARKS:
+              region_colors.push_back(kGreen);
+              break;
+            case SignalType::HUMAN:
+              region_colors.push_back(kLightYellow);
+              break;
+            case SignalType::CAR:
+              region_colors.push_back(kMagenta);
+              break;
+            case SignalType::PET:
+              region_colors.push_back(kYellow);
+              break;
+            case SignalType::OBJECT:
+              region_colors.push_back(kCyan);
+              break;
+            case SignalType::MOTION:
+            case SignalType::TEXT:
+            case SignalType::LOGO:
+              region_colors.push_back(kRed);
+              break;
+            case SignalType::USER_HINT:
+            default:
+              region_colors.push_back(kWhite);
+              break;
+          }
+        } else {
+          // For the case where "custom" signal type is used.
+          region_colors.push_back(kWhite);
+        }
+      }
+      key_frame_idx++;
+    }
+
+    cv::rectangle(viz_mat, crop_corners.first, crop_corners.second, kGreen, 4);
+    for (int j = 0; j < region_corners.size(); ++j) {
+      cv::rectangle(viz_mat, region_corners[j].first, region_corners[j].second,
+                    region_colors[j], 2);
+    }
+    viz_frames->push_back(std::move(viz_frame));
+  }
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status DrawFocusPointAndCropWindow(
+    const std::vector<cv::Mat>& scene_frames,
+    const std::vector<FocusPointFrame>& focus_point_frames,
+    const float overlay_opacity, const int crop_window_width,
+    const int crop_window_height, const ImageFormat::Format image_format,
+    std::vector<std::unique_ptr<ImageFrame>>* viz_frames) {
+  RET_CHECK(viz_frames) << "Output viz frames is null.";
+  viz_frames->clear();
+  const int num_frames = scene_frames.size();
+  RET_CHECK_GT(crop_window_width, 0) << "Crop window width is non-positive.";
+  RET_CHECK_GT(crop_window_height, 0) << "Crop window height is non-positive.";
+  const int half_width = crop_window_width / 2;
+  const int half_height = crop_window_height / 2;
+
+  for (int i = 0; i < num_frames; ++i) {
+    const auto& scene_frame = scene_frames[i];
+    auto viz_frame = absl::make_unique<ImageFrame>(
+        image_format, scene_frame.cols, scene_frame.rows);
+    cv::Mat darkened = formats::MatView(viz_frame.get());
+    scene_frame.copyTo(darkened);
+    cv::Mat viz_mat = darkened.clone();
+
+    // Darken the background.
+    cv::Mat overlay = cv::Mat::zeros(darkened.size(), darkened.type());
+    cv::addWeighted(overlay, overlay_opacity, darkened, 1 - overlay_opacity, 0,
+                    darkened);
+
+    if (focus_point_frames[i].point_size() > 0) {
+      float center_x = 0.0f, center_y = 0.0f;
+      for (int j = 0; j < focus_point_frames[i].point_size(); ++j) {
+        const auto& point = focus_point_frames[i].point(j);
+        const int x = point.norm_point_x() * scene_frame.cols;
+        const int y = point.norm_point_y() * scene_frame.rows;
+        cv::circle(viz_mat, cv::Point(x, y), 3, kRed, CV_FILLED);
+        center_x += x;
+        center_y += y;
+      }
+      center_x /= focus_point_frames[i].point_size();
+      center_y /= focus_point_frames[i].point_size();
+      cv::Point min_corner(center_x - half_width, center_y - half_height);
+      cv::Point max_corner(center_x + half_width, center_y + half_height);
+      viz_mat(cv::Rect(min_corner, max_corner))
+          .copyTo(darkened(cv::Rect(min_corner, max_corner)));
+    }
+    viz_frames->push_back(std::move(viz_frame));
+  }
+  return ::mediapipe::OkStatus();
+}
+
+}  // namespace autoflip
+}  // namespace mediapipe

mediapipe/examples/desktop/autoflip/quality/scene_cropping_viz.h

@@ -0,0 +1,61 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/examples/desktop/autoflip/quality/focus_point.pb.h"
+#ifndef MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_SCENE_CROPPING_VIZ_H_
+#define MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_SCENE_CROPPING_VIZ_H_
+
+#include <vector>
+
+#include "mediapipe/examples/desktop/autoflip/quality/cropping.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/image_format.pb.h"
+#include "mediapipe/framework/formats/image_frame.h"
+#include "mediapipe/framework/port/opencv_core_inc.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+namespace autoflip {
+
+// Draws the detections and crop regions on the scene frame. To make
+// visualization smoother, applies piecewise-constant interpolation on non-key
+// frames. This helps visualize the inputs to and outputs from the
+// FrameCropRegionComputer. Uses thick green for computed crop regions. Uses
+// different colors for different focus signals, faces are green, motion is
+// magenta, logos are red, ocrs are yellow (foreground) and light yellow
+// (background), brain objects are cyan, ica objects are orange, and the rest
+// are white.
+::mediapipe::Status DrawDetectionsAndCropRegions(
+    const std::vector<cv::Mat>& scene_frames,
+    const std::vector<bool>& is_key_frames,
+    const std::vector<KeyFrameInfo>& key_frame_infos,
+    const std::vector<KeyFrameCropResult>& key_frame_crop_results,
+    const mediapipe::ImageFormat::Format image_format,
+    std::vector<std::unique_ptr<ImageFrame>>* viz_frames);
+
+// Draws the focus point from the given FocusPointFrame and the crop window
+// centered around it on the scene frame in red. This helps visualize the input
+// to the retargeter.
+::mediapipe::Status DrawFocusPointAndCropWindow(
+    const std::vector<cv::Mat>& scene_frames,
+    const std::vector<FocusPointFrame>& focus_point_frames,
+    const float overlay_opacity, const int crop_window_width,
+    const int crop_window_height,
+    const mediapipe::ImageFormat::Format image_format,
+    std::vector<std::unique_ptr<ImageFrame>>* viz_frames);
+
+}  // namespace autoflip
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_SCENE_CROPPING_VIZ_H_

mediapipe/examples/desktop/autoflip/quality/testdata/BUILD

@@ -0,0 +1,19 @@
+# Copyright 2019 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+licenses(["notice"])  # Apache 2.0
+
+package(default_visibility = ["//visibility:public"])
+
+exports_files(glob(["*"]))

mediapipe/examples/desktop/autoflip/quality/testdata/camera_motion_tracking_scene_frame_results.csv

@@ -0,0 +1,30 @@
+0.14,0.6,5.00005
+0.125,0.583333,12.5
+0.11,0.566667,20
+0.0950004,0.55,27.5
+0.0800006,0.533334,35
+0.0650008,0.516668,42.5
+0.0500009,0.500001,50
+0.108329,0.549996,58.3333
+0.166662,0.599996,66.6667
+0.224995,0.649996,75
+0.283327,0.699995,83.3333
+0.34166,0.749995,91.6667
+0.399993,0.799994,100
+0.449994,0.666684,88.3357
+0.499993,0.533352,76.6691
+0.549993,0.40002,65.0024
+0.599992,0.266688,53.3357
+0.649992,0.133356,41.6691
+0.699991,2.40E-05,30.0024
+0.633346,0.033327,32.4999
+0.56668,0.06666,34.9999
+0.500014,0.099993,37.4999
+0.433348,0.133326,39.9998
+0.366682,0.166659,42.4998
+0.300016,0.199992,44.9999
+0.3,0.2,45.0005
+0.3,0.2,45.0005
+0.3,0.2,45.0005
+0.3,0.2,45.0005
+0.3,0.2,45.0005

mediapipe/examples/desktop/autoflip/quality/utils.cc

@@ -0,0 +1,451 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/examples/desktop/autoflip/quality/utils.h"
+
+#include <math.h>
+
+#include <algorithm>
+#include <utility>
+
+#include "absl/memory/memory.h"
+#include "mediapipe/examples/desktop/autoflip/quality/math_utils.h"
+#include "mediapipe/framework/port/opencv_imgproc_inc.h"
+#include "mediapipe/framework/port/ret_check.h"
+
+namespace mediapipe {
+namespace autoflip {
+namespace {
+
+// Returns true if the first pair should be considered greater than the second.
+// This is used to sort detections by scores (from high to low).
+bool PairCompare(const std::pair<float, int>& pair1,
+                 const std::pair<float, int>& pair2) {
+  return pair1.first > pair2.first;
+}
+
+}  // namespace
+
+template <typename T>
+void ScaleRect(const T& original_location, const double scale_x,
+               const double scale_y, Rect* scaled_location) {
+  scaled_location->set_x(round(original_location.x() * scale_x));
+  scaled_location->set_y(round(original_location.y() * scale_y));
+  scaled_location->set_width(round(original_location.width() * scale_x));
+  scaled_location->set_height(round(original_location.height() * scale_y));
+}
+template void ScaleRect<Rect>(const Rect&, const double, const double, Rect*);
+template void ScaleRect<RectF>(const RectF&, const double, const double, Rect*);
+
+void NormalizedRectToRect(const RectF& normalized_location, const int width,
+                          const int height, Rect* location) {
+  ScaleRect(normalized_location, width, height, location);
+}
+
+::mediapipe::Status ClampRect(const int width, const int height,
+                              Rect* location) {
+  return ClampRect(0, 0, width, height, location);
+}
+
+::mediapipe::Status ClampRect(const int x0, const int y0, const int x1,
+                              const int y1, Rect* location) {
+  RET_CHECK(!(location->x() >= x1 || location->x() + location->width() <= x0 ||
+              location->y() >= y1 || location->y() + location->height() <= y0));
+
+  int clamped_left, clamped_right, clamped_top, clamped_bottom;
+  RET_CHECK(MathUtil::Clamp(x0, x1, location->x(), &clamped_left));
+  RET_CHECK(MathUtil::Clamp(x0, x1, location->x() + location->width(),
+                            &clamped_right));
+  RET_CHECK(MathUtil::Clamp(y0, y1, location->y(), &clamped_top));
+  RET_CHECK(MathUtil::Clamp(y0, y1, location->y() + location->height(),
+                            &clamped_bottom));
+  location->set_x(clamped_left);
+  location->set_y(clamped_top);
+  location->set_width(std::max(0, clamped_right - clamped_left));
+  location->set_height(std::max(0, clamped_bottom - clamped_top));
+  return ::mediapipe::OkStatus();
+}
+
+void RectUnion(const Rect& rect_to_add, Rect* rect) {
+  const int x1 = std::min(rect->x(), rect_to_add.x());
+  const int y1 = std::min(rect->y(), rect_to_add.y());
+  const int x2 = std::max(rect->x() + rect->width(),
+                          rect_to_add.x() + rect_to_add.width());
+  const int y2 = std::max(rect->y() + rect->height(),
+                          rect_to_add.y() + rect_to_add.height());
+  rect->set_x(x1);
+  rect->set_y(y1);
+  rect->set_width(x2 - x1);
+  rect->set_height(y2 - y1);
+}
+
+::mediapipe::Status PackKeyFrameInfo(const int64 frame_timestamp_ms,
+                                     const DetectionSet& detections,
+                                     const int original_frame_width,
+                                     const int original_frame_height,
+                                     const int feature_frame_width,
+                                     const int feature_frame_height,
+                                     KeyFrameInfo* key_frame_info) {
+  RET_CHECK(key_frame_info != nullptr) << "KeyFrameInfo is null";
+  RET_CHECK(original_frame_width > 0 && original_frame_height > 0 &&
+            feature_frame_width > 0 && feature_frame_height > 0)
+      << "Invalid frame size.";
+
+  const double scale_x =
+      static_cast<double>(original_frame_width) / feature_frame_width;
+  const double scale_y =
+      static_cast<double>(original_frame_height) / feature_frame_height;
+
+  key_frame_info->set_timestamp_ms(frame_timestamp_ms);
+
+  // Scales detections and filter out the ones with no bounding boxes.
+  auto* processed_detections = key_frame_info->mutable_detections();
+  for (const auto& original_detection : detections.detections()) {
+    bool has_valid_location = true;
+    Rect location;
+    if (original_detection.has_location_normalized()) {
+      NormalizedRectToRect(original_detection.location_normalized(),
+                           original_frame_width, original_frame_height,
+                           &location);
+    } else if (original_detection.has_location()) {
+      ScaleRect(original_detection.location(), scale_x, scale_y, &location);
+    } else {
+      has_valid_location = false;
+      LOG(ERROR) << "Detection missing a bounding box, skipped.";
+    }
+    if (has_valid_location) {
+      auto* detection = processed_detections->add_detections();
+      *detection = original_detection;
+      RET_CHECK_OK(
+          ClampRect(original_frame_width, original_frame_height, &location));
+      *(detection->mutable_location()) = location;
+    }
+  }
+
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status SortDetections(
+    const DetectionSet& detections,
+    std::vector<SalientRegion>* required_regions,
+    std::vector<SalientRegion>* non_required_regions) {
+  required_regions->clear();
+  non_required_regions->clear();
+
+  // Makes pairs of score and index.
+  std::vector<std::pair<float, int>> required_score_idx_pairs;
+  std::vector<std::pair<float, int>> non_required_score_idx_pairs;
+  for (int i = 0; i < detections.detections_size(); ++i) {
+    const auto& detection = detections.detections(i);
+    const auto pair = std::make_pair(detection.score(), i);
+    if (detection.is_required()) {
+      required_score_idx_pairs.push_back(pair);
+    } else {
+      non_required_score_idx_pairs.push_back(pair);
+    }
+  }
+
+  // Sorts required regions by score.
+  std::stable_sort(required_score_idx_pairs.begin(),
+                   required_score_idx_pairs.end(), PairCompare);
+  for (int i = 0; i < required_score_idx_pairs.size(); ++i) {
+    const int original_idx = required_score_idx_pairs[i].second;
+    required_regions->push_back(detections.detections(original_idx));
+  }
+
+  // Sorts non-required regions by score.
+  std::stable_sort(non_required_score_idx_pairs.begin(),
+                   non_required_score_idx_pairs.end(), PairCompare);
+  for (int i = 0; i < non_required_score_idx_pairs.size(); ++i) {
+    const int original_idx = non_required_score_idx_pairs[i].second;
+    non_required_regions->push_back(detections.detections(original_idx));
+  }
+
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status SetKeyFrameCropTarget(const int frame_width,
+                                          const int frame_height,
+                                          const double target_aspect_ratio,
+                                          KeyFrameCropOptions* crop_options) {
+  RET_CHECK_NE(crop_options, nullptr) << "KeyFrameCropOptions is null.";
+  RET_CHECK_GT(frame_width, 0) << "Frame width is non-positive.";
+  RET_CHECK_GT(frame_height, 0) << "Frame height is non-positive.";
+  RET_CHECK_GT(target_aspect_ratio, 0)
+      << "Target aspect ratio is non-positive.";
+  const double input_aspect_ratio =
+      static_cast<double>(frame_width) / frame_height;
+  const int crop_target_width =
+      target_aspect_ratio < input_aspect_ratio
+          ? std::round(frame_height * target_aspect_ratio)
+          : frame_width;
+  const int crop_target_height =
+      target_aspect_ratio < input_aspect_ratio
+          ? frame_height
+          : std::round(frame_width / target_aspect_ratio);
+  crop_options->set_target_width(crop_target_width);
+  crop_options->set_target_height(crop_target_height);
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status AggregateKeyFrameResults(
+    const std::vector<KeyFrameInfo>& key_frame_infos,
+    const KeyFrameCropOptions& key_frame_crop_options,
+    const std::vector<KeyFrameCropResult>& key_frame_crop_results,
+    const int scene_frame_width, const int scene_frame_height,
+    SceneKeyFrameCropSummary* scene_summary) {
+  RET_CHECK_NE(scene_summary, nullptr)
+      << "Output SceneKeyFrameCropSummary is null.";
+
+  const int num_key_frames = key_frame_infos.size();
+  RET_CHECK_EQ(num_key_frames, key_frame_crop_results.size())
+      << "Inconsistent number of key frames:"
+      << " num_key_frames = " << num_key_frames
+      << " key_frame_crop_results.size() = " << key_frame_crop_results.size();
+
+  RET_CHECK_GT(scene_frame_width, 0) << "Non-positive frame width.";
+  RET_CHECK_GT(scene_frame_height, 0) << "Non-positive frame height.";
+
+  const int target_width = key_frame_crop_options.target_width();
+  const int target_height = key_frame_crop_options.target_height();
+  RET_CHECK_GT(target_width, 0) << "Non-positive target width.";
+  RET_CHECK_GT(target_height, 0) << "Non-positive target height.";
+  RET_CHECK_LE(target_width, scene_frame_width)
+      << "Target width exceeds frame width.";
+  RET_CHECK_LE(target_height, scene_frame_height)
+      << "Target height exceeds frame height.";
+
+  scene_summary->set_scene_frame_width(scene_frame_width);
+  scene_summary->set_scene_frame_height(scene_frame_height);
+  scene_summary->set_crop_window_width(target_width);
+  scene_summary->set_crop_window_height(target_height);
+
+  // Handles the corner case of no key frames.
+  if (num_key_frames == 0) {
+    scene_summary->set_has_salient_region(false);
+    return ::mediapipe::OkStatus();
+  }
+
+  scene_summary->set_num_key_frames(num_key_frames);
+  scene_summary->set_key_frame_center_min_x(scene_frame_width);
+  scene_summary->set_key_frame_center_max_x(0);
+  scene_summary->set_key_frame_center_min_y(scene_frame_height);
+  scene_summary->set_key_frame_center_max_y(0);
+  scene_summary->set_key_frame_min_score(std::numeric_limits<float>::max());
+  scene_summary->set_key_frame_max_score(0.0);
+
+  const float half_height = target_height / 2.0f;
+  const float half_width = target_width / 2.0f;
+  bool has_salient_region = false;
+  int num_success_frames = 0;
+  std::unique_ptr<Rect> required_crop_region_union = nullptr;
+  for (int i = 0; i < num_key_frames; ++i) {
+    auto* key_frame_compact_info = scene_summary->add_key_frame_compact_infos();
+    key_frame_compact_info->set_timestamp_ms(key_frame_infos[i].timestamp_ms());
+    const auto& result = key_frame_crop_results[i];
+    if (result.are_required_regions_covered_in_target_size()) {
+      num_success_frames++;
+    }
+    if (result.region_is_empty()) {
+      key_frame_compact_info->set_center_x(-1.0);
+      key_frame_compact_info->set_center_y(-1.0);
+      key_frame_compact_info->set_score(-1.0);
+      continue;
+    }
+
+    has_salient_region = true;
+    if (!result.required_region_is_empty()) {
+      if (required_crop_region_union == nullptr) {
+        required_crop_region_union =
+            absl::make_unique<Rect>(result.required_region());
+      } else {
+        RectUnion(result.required_region(), required_crop_region_union.get());
+      }
+    }
+
+    const auto& region = result.region();
+    float original_center_x = region.x() + region.width() / 2.0f;
+    float original_center_y = region.y() + region.height() / 2.0f;
+    RET_CHECK_GE(original_center_x, 0) << "Negative horizontal center.";
+    RET_CHECK_GE(original_center_y, 0) << "Negative vertical center.";
+    // Ensure that centered region of target size does not exceed frame size.
+    float center_x, center_y;
+    RET_CHECK(MathUtil::Clamp(half_width, scene_frame_width - half_width,
+                              original_center_x, &center_x));
+    RET_CHECK(MathUtil::Clamp(half_height, scene_frame_height - half_height,
+                              original_center_y, &center_y));
+    key_frame_compact_info->set_center_x(center_x);
+    key_frame_compact_info->set_center_y(center_y);
+    scene_summary->set_key_frame_center_min_x(
+        std::min(scene_summary->key_frame_center_min_x(), center_x));
+    scene_summary->set_key_frame_center_max_x(
+        std::max(scene_summary->key_frame_center_max_x(), center_x));
+    scene_summary->set_key_frame_center_min_y(
+        std::min(scene_summary->key_frame_center_min_y(), center_y));
+    scene_summary->set_key_frame_center_max_y(
+        std::max(scene_summary->key_frame_center_max_y(), center_y));
+
+    scene_summary->set_crop_window_width(
+        std::max(scene_summary->crop_window_width(), region.width()));
+    scene_summary->set_crop_window_height(
+        std::max(scene_summary->crop_window_height(), region.height()));
+
+    const float score = result.region_score();
+    RET_CHECK_GE(score, 0.0) << "Negative score.";
+    key_frame_compact_info->set_score(result.region_score());
+    scene_summary->set_key_frame_min_score(
+        std::min(scene_summary->key_frame_min_score(), score));
+    scene_summary->set_key_frame_max_score(
+        std::max(scene_summary->key_frame_max_score(), score));
+  }
+
+  scene_summary->set_has_salient_region(has_salient_region);
+  scene_summary->set_has_required_salient_region(required_crop_region_union !=
+                                                 nullptr);
+  if (required_crop_region_union) {
+    *(scene_summary->mutable_key_frame_required_crop_region_union()) =
+        *required_crop_region_union;
+  }
+  const float success_rate =
+      static_cast<float>(num_success_frames) / num_key_frames;
+  scene_summary->set_frame_success_rate(success_rate);
+  const float motion_x =
+      static_cast<float>(scene_summary->key_frame_center_max_x() -
+                         scene_summary->key_frame_center_min_x()) /
+      scene_frame_width;
+  scene_summary->set_horizontal_motion_amount(motion_x);
+  const float motion_y =
+      static_cast<float>(scene_summary->key_frame_center_max_y() -
+                         scene_summary->key_frame_center_min_y()) /
+      scene_frame_height;
+  scene_summary->set_vertical_motion_amount(motion_y);
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status ComputeSceneStaticBordersSize(
+    const std::vector<StaticFeatures>& static_features, int* top_border_size,
+    int* bottom_border_size) {
+  RET_CHECK(top_border_size) << "Output top border size is null.";
+  RET_CHECK(bottom_border_size) << "Output bottom border size is null.";
+
+  *top_border_size = -1;
+  for (int i = 0; i < static_features.size(); ++i) {
+    bool has_static_top_border = false;
+    for (const auto& feature : static_features[i].border()) {
+      if (feature.relative_position() == Border::TOP) {
+        has_static_top_border = true;
+        const int static_size = feature.border_position().height();
+        *top_border_size = (*top_border_size > 0)
+                               ? std::min(*top_border_size, static_size)
+                               : static_size;
+      }
+    }
+    if (!has_static_top_border) {
+      *top_border_size = 0;
+      break;
+    }
+  }
+
+  *bottom_border_size = -1;
+  for (int i = 0; i < static_features.size(); ++i) {
+    bool has_static_bottom_border = false;
+    for (const auto& feature : static_features[i].border()) {
+      if (feature.relative_position() == Border::BOTTOM) {
+        has_static_bottom_border = true;
+        const int static_size = feature.border_position().height();
+        *bottom_border_size = (*bottom_border_size > 0)
+                                  ? std::min(*bottom_border_size, static_size)
+                                  : static_size;
+      }
+    }
+    if (!has_static_bottom_border) {
+      *bottom_border_size = 0;
+      break;
+    }
+  }
+
+  *top_border_size = std::max(0, *top_border_size);
+  *bottom_border_size = std::max(0, *bottom_border_size);
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status FindSolidBackgroundColor(
+    const std::vector<StaticFeatures>& static_features,
+    const std::vector<int64>& static_features_timestamps,
+    const double min_fraction_solid_background_color,
+    bool* has_solid_background,
+    PiecewiseLinearFunction* background_color_l_function,
+    PiecewiseLinearFunction* background_color_a_function,
+    PiecewiseLinearFunction* background_color_b_function) {
+  RET_CHECK(has_solid_background) << "Output boolean is null.";
+  RET_CHECK(background_color_l_function) << "Output color l function is null.";
+  RET_CHECK(background_color_a_function) << "Output color a function is null.";
+  RET_CHECK(background_color_b_function) << "Output color b function is null.";
+
+  *has_solid_background = false;
+  int solid_background_frames = 0;
+  for (int i = 0; i < static_features.size(); ++i) {
+    if (static_features[i].has_solid_background()) {
+      solid_background_frames++;
+      const auto& color = static_features[i].solid_background();
+      const int64 timestamp = static_features_timestamps[i];
+      // BorderDetectionCalculator sets color assuming the input frame is
+      // BGR, but in reality we have RGB, so we need to revert it here.
+      // TODO remove this custom logic in BorderDetectionCalculator,
+      // original CroppingCalculator, and this calculator.
+      cv::Mat3f rgb_mat(1, 1, cv::Vec3b(color.b(), color.g(), color.r()));
+      // Necessary scaling of the RGB values from [0, 255] to [0, 1] based on:
+      // https://docs.opencv.org/2.4/modules/imgproc/doc/miscellaneous_transformations.html#cvtcolor
+      rgb_mat *= 1.0 / 255;
+      cv::Mat3f lab_mat(1, 1);
+      cv::cvtColor(rgb_mat, lab_mat, cv::COLOR_RGB2Lab);
+      // TODO change to piecewise constant interpolation if there is
+      // visual artifact. We can simply add one more point right before the
+      // next point with same value to mimic piecewise constant behavior.
+      const auto lab = lab_mat.at<cv::Vec3f>(0, 0);
+      background_color_l_function->AddPoint(timestamp, lab[0]);
+      background_color_a_function->AddPoint(timestamp, lab[1]);
+      background_color_b_function->AddPoint(timestamp, lab[2]);
+    }
+  }
+
+  if (!static_features.empty() &&
+      static_cast<float>(solid_background_frames) / static_features.size() >=
+          min_fraction_solid_background_color) {
+    *has_solid_background = true;
+  }
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status AffineRetarget(
+    const cv::Size& output_size, const std::vector<cv::Mat>& frames,
+    const std::vector<cv::Mat>& affine_projection,
+    std::vector<cv::Mat>* cropped_frames) {
+  RET_CHECK(frames.size() == affine_projection.size())
+      << "number of frames and retarget offsets must be the same.";
+  RET_CHECK(cropped_frames->size() == frames.size())
+      << "Output vector cropped_frames must be populated with output images of "
+         "the same type, size and count.";
+  for (int i = 0; i < frames.size(); i++) {
+    RET_CHECK(frames[i].type() == (*cropped_frames)[i].type())
+        << "input and output images must be the same type.";
+    const auto affine = affine_projection[i];
+    RET_CHECK(affine.cols == 3) << "Affine matrix must be 2x3";
+    RET_CHECK(affine.rows == 2) << "Affine matrix must be 2x3";
+    cv::warpAffine(frames[i], (*cropped_frames)[i], affine, output_size);
+  }
+  return ::mediapipe::OkStatus();
+}
+}  // namespace autoflip
+}  // namespace mediapipe

mediapipe/examples/desktop/autoflip/quality/utils.h

@@ -0,0 +1,119 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_UTILS_H_
+#define MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_UTILS_H_
+
+#include <vector>
+
+#include "mediapipe/examples/desktop/autoflip/autoflip_messages.pb.h"
+#include "mediapipe/examples/desktop/autoflip/quality/cropping.pb.h"
+#include "mediapipe/examples/desktop/autoflip/quality/piecewise_linear_function.h"
+#include "mediapipe/framework/port/opencv_core_inc.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+namespace autoflip {
+
+// Packs detected features and timestamp (ms) into a KeyFrameInfo object. Scales
+// features back to the original frame size if features have been detected on a
+// different frame size.
+::mediapipe::Status PackKeyFrameInfo(const int64 frame_timestamp_ms,
+                                     const DetectionSet& detections,
+                                     const int original_frame_width,
+                                     const int original_frame_height,
+                                     const int feature_frame_width,
+                                     const int feature_frame_height,
+                                     KeyFrameInfo* key_frame_info);
+
+// Sorts required and non-required salient regions given a detection set.
+::mediapipe::Status SortDetections(
+    const DetectionSet& detections,
+    std::vector<SalientRegion>* required_regions,
+    std::vector<SalientRegion>* non_required_regions);
+
+// Sets the target crop size in KeyFrameCropOptions based on frame size and
+// target aspect ratio so that the target crop size covers the biggest area
+// possible in the frame.
+::mediapipe::Status SetKeyFrameCropTarget(const int frame_width,
+                                          const int frame_height,
+                                          const double target_aspect_ratio,
+                                          KeyFrameCropOptions* crop_options);
+
+// Aggregates information from KeyFrameInfos and KeyFrameCropResults into
+// SceneKeyFrameCropSummary.
+::mediapipe::Status AggregateKeyFrameResults(
+    const std::vector<KeyFrameInfo>& key_frame_infos,
+    const KeyFrameCropOptions& key_frame_crop_options,
+    const std::vector<KeyFrameCropResult>& key_frame_crop_results,
+    const int scene_frame_width, const int scene_frame_height,
+    SceneKeyFrameCropSummary* scene_summary);
+
+// Computes the static top and border size across a scene given a vector of
+// StaticFeatures over frames.
+::mediapipe::Status ComputeSceneStaticBordersSize(
+    const std::vector<StaticFeatures>& static_features, int* top_border_size,
+    int* bottom_border_size);
+
+// Finds the solid background colors in a scene from input StaticFeatures.
+// Sets has_solid_background to true if the number of frames with solid
+// background color exceeds given threshold, i.e.,
+// min_fraction_solid_background_color. Builds the background color
+// interpolation functions in Lab space using input timestamps.
+::mediapipe::Status FindSolidBackgroundColor(
+    const std::vector<StaticFeatures>& static_features,
+    const std::vector<int64>& static_features_timestamps,
+    const double min_fraction_solid_background_color,
+    bool* has_solid_background,
+    PiecewiseLinearFunction* background_color_l_function,
+    PiecewiseLinearFunction* background_color_a_function,
+    PiecewiseLinearFunction* background_color_b_function);
+
+// Helpers to scale, clamp, and take union of rectangles. These functions do not
+// check for pointers not being null or rectangles being valid.
+
+// Scales a rectangle given horizontal and vertical scaling factors.
+template <typename T>
+void ScaleRect(const T& original_location, const double scale_x,
+               const double scale_y, Rect* scaled_location);
+
+// Converts a normalized rectangle to a rectangle given width and height.
+void NormalizedRectToRect(const RectF& normalized_location, const int width,
+                          const int height, Rect* location);
+
+// Clamps a rectangle to lie within [x0, y0] and [x1, y1]. Returns true if the
+// rectangle has any overlapping with the target window.
+::mediapipe::Status ClampRect(const int x0, const int y0, const int x1,
+                              const int y1, Rect* location);
+
+// Convenience function to clamp a rectangle to lie within [0, 0] and
+// [width, height].
+::mediapipe::Status ClampRect(const int width, const int height,
+                              Rect* location);
+
+// Enlarges a given rectangle to cover a new rectangle to be added.
+void RectUnion(const Rect& rect_to_add, Rect* rect);
+
+// Performs an affine retarget on a list of input images.  Output vector
+// cropped_frames must be filled with Mats of the same size as output_size and
+// type.
+::mediapipe::Status AffineRetarget(
+    const cv::Size& output_size, const std::vector<cv::Mat>& frames,
+    const std::vector<cv::Mat>& affine_projection,
+    std::vector<cv::Mat>* cropped_frames);
+
+}  // namespace autoflip
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_UTILS_H_

mediapipe/examples/desktop/autoflip/quality/visual_scorer.cc

@@ -0,0 +1,182 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/examples/desktop/autoflip/quality/visual_scorer.h"
+
+#include <math.h>
+
+#include <algorithm>
+#include <cmath>
+#include <memory>
+#include <vector>
+
+#include "mediapipe/framework/port/opencv_core_inc.h"
+#include "mediapipe/framework/port/opencv_imgproc_inc.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/port/status_builder.h"
+
+// Weight threshold for computing a value.
+constexpr float kEpsilon = 0.0001;
+
+namespace mediapipe {
+namespace autoflip {
+namespace {
+
+// Crop the given rectangle so that it fits in the given 2D matrix.
+void CropRectToMat(const cv::Mat& image, cv::Rect* rect) {
+  int x = std::min(std::max(rect->x, 0), image.cols);
+  int y = std::min(std::max(rect->y, 0), image.rows);
+  int w = std::min(std::max(rect->x + rect->width, 0), image.cols) - x;
+  int h = std::min(std::max(rect->y + rect->height, 0), image.rows) - y;
+  *rect = cv::Rect(x, y, w, h);
+}
+
+}  // namespace
+
+VisualScorer::VisualScorer(const VisualScorerOptions& options)
+    : options_(options) {}
+
+mediapipe::Status VisualScorer::CalculateScore(const cv::Mat& image,
+                                               const SalientRegion& region,
+                                               float* score) const {
+  const float weight_sum = options_.area_weight() +
+                           options_.sharpness_weight() +
+                           options_.colorfulness_weight();
+
+  // Crop the region to fit in the image.
+  cv::Rect region_rect;
+  if (region.has_location()) {
+    region_rect =
+        cv::Rect(region.location().x(), region.location().y(),
+                 region.location().width(), region.location().height());
+  } else if (region.has_location_normalized()) {
+    region_rect = cv::Rect(region.location_normalized().x() * image.cols,
+                           region.location_normalized().y() * image.rows,
+                           region.location_normalized().width() * image.cols,
+                           region.location_normalized().height() * image.rows);
+  } else {
+    return ::mediapipe::UnknownErrorBuilder(MEDIAPIPE_LOC)
+           << "Unset region location.";
+  }
+
+  CropRectToMat(image, &region_rect);
+  if (region_rect.area() == 0) {
+    *score = 0;
+    return ::mediapipe::OkStatus();
+  }
+
+  // Compute a score based on area covered by this region.
+  const float area_score =
+      options_.area_weight() * region_rect.area() / (image.cols * image.rows);
+
+  // Convert the visible region to cv::Mat.
+  cv::Mat image_region_mat = image(region_rect);
+
+  // Compute a score from sharpness.
+
+  float sharpness_score_result = 0.0;
+  if (options_.sharpness_weight() > kEpsilon) {
+    // TODO: implement a sharpness score or remove this code block.
+    return ::mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
+           << "sharpness scorer is not yet implemented, please set weight to "
+              "0.0";
+  }
+  const float sharpness_score =
+      options_.sharpness_weight() * sharpness_score_result;
+
+  // Compute a colorfulness score.
+  float colorfulness_score = 0;
+  if (options_.colorfulness_weight() > kEpsilon) {
+    MP_RETURN_IF_ERROR(
+        CalculateColorfulness(image_region_mat, &colorfulness_score));
+    colorfulness_score *= options_.colorfulness_weight();
+  }
+
+  *score = (area_score + sharpness_score + colorfulness_score) / weight_sum;
+  if (*score > 1.0f || *score < 0.0f) {
+    LOG(WARNING) << "Score of region outside expected range: " << *score;
+  }
+  return ::mediapipe::OkStatus();
+}
+
+mediapipe::Status VisualScorer::CalculateColorfulness(
+    const cv::Mat& image, float* colorfulness) const {
+  // Convert the image to HSV.
+  cv::Mat image_hsv;
+  cv::cvtColor(image, image_hsv, CV_RGB2HSV);
+
+  // Mask out pixels that are too dark or too bright.
+  cv::Mat mask(image.rows, image.cols, CV_8UC1);
+  bool empty_mask = true;
+  for (int x = 0; x < image.cols; ++x) {
+    for (int y = 0; y < image.rows; ++y) {
+      const cv::Vec3b& pixel = image.at<cv::Vec3b>(x, y);
+      const bool is_usable =
+          (std::min(pixel.val[0], std::min(pixel.val[1], pixel.val[2])) < 250 &&
+           std::max(pixel.val[0], std::max(pixel.val[1], pixel.val[2])) > 5);
+      mask.at<unsigned char>(y, x) = is_usable ? 255 : 0;
+      if (is_usable) empty_mask = false;
+    }
+  }
+
+  // If the mask is empty, return.
+  if (empty_mask) {
+    *colorfulness = 0;
+    return ::mediapipe::OkStatus();
+  }
+
+  // Generate a 2D histogram (hue/saturation).
+  cv::MatND hs_histogram;
+  const int kHueBins = 10, kSaturationBins = 8;
+  const int kHistogramChannels[] = {0, 1};
+  const int kHistogramBinNum[] = {kHueBins, kSaturationBins};
+  const float kHueRange[] = {0, 180};
+  const float kSaturationRange[] = {0, 256};
+  const float* kHistogramRange[] = {kHueRange, kSaturationRange};
+  cv::calcHist(&image_hsv, 1, kHistogramChannels, mask, hs_histogram,
+               2 /* histogram dims */, kHistogramBinNum, kHistogramRange,
+               true /* uniform */, false /* accumulate */);
+
+  // Convert to a hue histogram and weigh saturated pixels more.
+  std::vector<float> hue_histogram(kHueBins, 0.0f);
+  float hue_sum = 0.0f;
+  for (int bin_s = 0; bin_s < kSaturationBins; ++bin_s) {
+    const float weight = std::pow(2.0f, bin_s);
+    for (int bin_h = 0; bin_h < kHueBins; ++bin_h) {
+      float value = hs_histogram.at<float>(bin_h, bin_s) * weight;
+      hue_histogram[bin_h] += value;
+      hue_sum += value;
+    }
+  }
+  if (hue_sum == 0.0f) {
+    *colorfulness = 0;
+    return ::mediapipe::OkStatus();
+  }
+
+  // Compute the histogram entropy.
+  *colorfulness = 0;
+  for (int bin = 0; bin < kHueBins; ++bin) {
+    float value = hue_histogram[bin] / hue_sum;
+    if (value > 0.0f) {
+      *colorfulness -= value * std::log(value);
+    }
+  }
+  *colorfulness /= std::log(2.0f);
+
+  return ::mediapipe::OkStatus();
+}
+
+}  // namespace autoflip
+}  // namespace mediapipe

mediapipe/examples/desktop/autoflip/quality/visual_scorer.h

@@ -0,0 +1,47 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_VISUAL_SCORER_H_
+#define MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_VISUAL_SCORER_H_
+
+#include "mediapipe/examples/desktop/autoflip/autoflip_messages.pb.h"
+#include "mediapipe/examples/desktop/autoflip/quality/visual_scorer.pb.h"
+#include "mediapipe/framework/port/opencv_core_inc.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+namespace autoflip {
+
+// This class scores a SalientRegion within an image based on weighted averages
+// of various signals computed on the patch.
+class VisualScorer {
+ public:
+  explicit VisualScorer(const VisualScorerOptions& options);
+
+  // Computes a score on a salientregion and returns a value [0...1].
+  mediapipe::Status CalculateScore(const cv::Mat& image,
+                                   const SalientRegion& region,
+                                   float* score) const;
+
+ private:
+  mediapipe::Status CalculateColorfulness(const cv::Mat& image,
+                                          float* colorfulness) const;
+
+  VisualScorerOptions options_;
+};
+
+}  // namespace autoflip
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_VISUAL_SCORER_H_

mediapipe/examples/desktop/autoflip/quality/visual_scorer.proto

@@ -0,0 +1,28 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe.autoflip;
+
+// Options for the VisualScorer module.
+// Next tag: 6
+message VisualScorerOptions {
+  // Weights for the various cues. A larger weight means that the corresponding
+  // cue will be of higher importance when generating the combined score.
+  optional float area_weight = 1 [default = 1.0];
+  // Sharpness is not yet implemented.
+  optional float sharpness_weight = 2 [default = 0.0];
+  optional float colorfulness_weight = 3 [default = 0.0];
+}

mediapipe/examples/desktop/autoflip/quality/visual_scorer_test.cc

@@ -0,0 +1,80 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/examples/desktop/autoflip/quality/visual_scorer.h"
+
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/opencv_core_inc.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/status_matchers.h"
+
+namespace mediapipe {
+namespace autoflip {
+namespace {
+
+TEST(VisualScorerTest, ScoresArea) {
+  cv::Mat image_mat(200, 200, CV_8UC3);
+  SalientRegion region = ParseTextProtoOrDie<SalientRegion>(
+      R"(location { x: 10 y: 10 width: 100 height: 100 })");
+
+  VisualScorerOptions options = ParseTextProtoOrDie<VisualScorerOptions>(
+      R"(area_weight: 1.0 sharpness_weight: 0 colorfulness_weight: 0)");
+  VisualScorer scorer(options);
+  float score = 0.0;
+  MP_EXPECT_OK(scorer.CalculateScore(image_mat, region, &score));
+  EXPECT_EQ(0.25, score);  // (100 * 100) / (200 * 200).
+}
+
+TEST(VisualScorerTest, ScoresSharpness) {
+  SalientRegion region = ParseTextProtoOrDie<SalientRegion>(
+      R"(location { x: 10 y: 10 width: 100 height: 100 })");
+
+  VisualScorerOptions options = ParseTextProtoOrDie<VisualScorerOptions>(
+      R"(area_weight: 0 sharpness_weight: 1.0 colorfulness_weight: 0)");
+  VisualScorer scorer(options);
+
+  // Compute the score of an empty image and an image with a rectangle.
+  cv::Mat image_mat(200, 200, CV_8UC3);
+  image_mat.setTo(cv::Scalar(0, 0, 0));
+  float score_rect = 0;
+  auto status = scorer.CalculateScore(image_mat, region, &score_rect);
+  EXPECT_EQ(status.code(), StatusCode::kInvalidArgument);
+}
+
+TEST(VisualScorerTest, ScoresColorfulness) {
+  SalientRegion region = ParseTextProtoOrDie<SalientRegion>(
+      R"(location { x: 10 y: 10 width: 50 height: 150 })");
+
+  VisualScorerOptions options = ParseTextProtoOrDie<VisualScorerOptions>(
+      R"(area_weight: 0 sharpness_weight: 0 colorfulness_weight: 1.0)");
+  VisualScorer scorer(options);
+
+  // Compute the scores of images with 1, 2 and 3 colors.
+  cv::Mat image_mat(200, 200, CV_8UC3);
+  image_mat.setTo(cv::Scalar(0, 0, 255));
+  float score_1c = 0, score_2c = 0, score_3c = 0;
+  MP_EXPECT_OK(scorer.CalculateScore(image_mat, region, &score_1c));
+  image_mat(cv::Rect(30, 30, 20, 20)).setTo(cv::Scalar(128, 0, 0));
+  MP_EXPECT_OK(scorer.CalculateScore(image_mat, region, &score_2c));
+  image_mat(cv::Rect(50, 50, 20, 20)).setTo(cv::Scalar(255, 128, 0));
+  MP_EXPECT_OK(scorer.CalculateScore(image_mat, region, &score_3c));
+  // Images with more colors should have a higher score.
+  EXPECT_LT(score_1c, score_2c);
+  EXPECT_LT(score_2c, score_3c);
+}
+
+}  // namespace
+}  // namespace autoflip
+}  // namespace mediapipe

mediapipe/examples/desktop/autoflip/subgraph/BUILD

@@ -0,0 +1,39 @@
+load("//mediapipe/framework/tool:mediapipe_graph.bzl", "mediapipe_simple_subgraph")
+
+# Copyright 2019 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+licenses(["notice"])  # Apache 2.0
+
+package(default_visibility = ["//mediapipe/examples:__subpackages__"])
+
+mediapipe_simple_subgraph(
+    name = "autoflip_face_detection_subgraph",
+    graph = "face_detection_subgraph.pbtxt",
+    register_as = "AutoFlipFaceDetectionSubgraph",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/graphs/face_detection:desktop_tflite_calculators",
+    ],
+)
+
+mediapipe_simple_subgraph(
+    name = "autoflip_object_detection_subgraph",
+    graph = "autoflip_object_detection_subgraph.pbtxt",
+    register_as = "AutoFlipObjectDetectionSubgraph",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/graphs/object_detection:desktop_tflite_calculators",
+    ],
+)

mediapipe/examples/desktop/autoflip/subgraph/autoflip_object_detection_subgraph.pbtxt

@@ -0,0 +1,126 @@
+# MediaPipe graph that performs object detection with TensorFlow Lite on CPU.
+
+input_stream: "VIDEO:input_video"
+output_stream: "DETECTIONS:output_detections"
+
+# Transforms the input image on CPU to a 320x320 image. To scale the image, by
+# default it uses the STRETCH scale mode that maps the entire input image to the
+# entire transformed image. As a result, image aspect ratio may be changed and
+# objects in the image may be deformed (stretched or squeezed), but the object
+# detection model used in this graph is agnostic to that deformation.
+node: {
+  calculator: "ImageTransformationCalculator"
+  input_stream: "IMAGE:input_video"
+  output_stream: "IMAGE:transformed_input_video"
+  options: {
+    [mediapipe.ImageTransformationCalculatorOptions.ext] {
+      output_width: 320
+      output_height: 320
+    }
+  }
+}
+
+# Converts the transformed input image on CPU into an image tensor stored as a
+# TfLiteTensor.
+node {
+  calculator: "TfLiteConverterCalculator"
+  input_stream: "IMAGE:transformed_input_video"
+  output_stream: "TENSORS:image_tensor"
+}
+
+# Runs a TensorFlow Lite model on CPU that takes an image tensor and outputs a
+# vector of tensors representing, for instance, detection boxes/keypoints and
+# scores.
+node {
+  calculator: "TfLiteInferenceCalculator"
+  input_stream: "TENSORS:image_tensor"
+  output_stream: "TENSORS:detection_tensors"
+  options: {
+    [mediapipe.TfLiteInferenceCalculatorOptions.ext] {
+      model_path: "mediapipe/models/ssdlite_object_detection.tflite"
+    }
+  }
+}
+
+# Generates a single side packet containing a vector of SSD anchors based on
+# the specification in the options.
+node {
+  calculator: "SsdAnchorsCalculator"
+  output_side_packet: "anchors"
+  options: {
+    [mediapipe.SsdAnchorsCalculatorOptions.ext] {
+      num_layers: 6
+      min_scale: 0.2
+      max_scale: 0.95
+      input_size_height: 320
+      input_size_width: 320
+      anchor_offset_x: 0.5
+      anchor_offset_y: 0.5
+      strides: 16
+      strides: 32
+      strides: 64
+      strides: 128
+      strides: 256
+      strides: 512
+      aspect_ratios: 1.0
+      aspect_ratios: 2.0
+      aspect_ratios: 0.5
+      aspect_ratios: 3.0
+      aspect_ratios: 0.3333
+      reduce_boxes_in_lowest_layer: true
+    }
+  }
+}
+
+# Decodes the detection tensors generated by the TensorFlow Lite model, based on
+# the SSD anchors and the specification in the options, into a vector of
+# detections. Each detection describes a detected object.
+node {
+  calculator: "TfLiteTensorsToDetectionsCalculator"
+  input_stream: "TENSORS:detection_tensors"
+  input_side_packet: "ANCHORS:anchors"
+  output_stream: "DETECTIONS:detections"
+  options: {
+    [mediapipe.TfLiteTensorsToDetectionsCalculatorOptions.ext] {
+      num_classes: 91
+      num_boxes: 2034
+      num_coords: 4
+      ignore_classes: 0
+      sigmoid_score: true
+      apply_exponential_on_box_size: true
+      x_scale: 10.0
+      y_scale: 10.0
+      h_scale: 5.0
+      w_scale: 5.0
+      min_score_thresh: 0.6
+    }
+  }
+}
+
+# Performs non-max suppression to remove excessive detections.
+node {
+  calculator: "NonMaxSuppressionCalculator"
+  input_stream: "detections"
+  output_stream: "filtered_detections"
+  options: {
+    [mediapipe.NonMaxSuppressionCalculatorOptions.ext] {
+      min_suppression_threshold: 0.4
+      max_num_detections: 5
+      overlap_type: INTERSECTION_OVER_UNION
+      return_empty_detections: true
+    }
+  }
+}
+
+# Maps detection label IDs to the corresponding label text. The label map is
+# provided in the label_map_path option.
+node {
+  calculator: "DetectionLabelIdToTextCalculator"
+  input_stream: "filtered_detections"
+  output_stream: "output_detections"
+  options: {
+    [mediapipe.DetectionLabelIdToTextCalculatorOptions.ext] {
+      label_map_path: "mediapipe/models/ssdlite_object_detection_labelmap.txt"
+    }
+  }
+}

mediapipe/examples/desktop/autoflip/subgraph/face_detection_subgraph.pbtxt

@@ -0,0 +1,136 @@
+# MediaPipe graph that performs face detection with TensorFlow Lite on CPU.
+
+input_stream: "VIDEO:input_video"
+output_stream: "DETECTIONS:output_detections"
+
+
+# Transforms the input image on CPU to a 128x128 image. To scale the input
+# image, the scale_mode option is set to FIT to preserve the aspect ratio,
+# resulting in potential letterboxing in the transformed image.
+node: {
+  calculator: "ImageTransformationCalculator"
+  input_stream: "IMAGE:input_video"
+  output_stream: "IMAGE:transformed_input_video_cpu"
+  output_stream: "LETTERBOX_PADDING:letterbox_padding"
+  options: {
+    [mediapipe.ImageTransformationCalculatorOptions.ext] {
+      output_width: 128
+      output_height: 128
+      scale_mode: FIT
+    }
+  }
+}
+
+# Converts the transformed input image on CPU into an image tensor stored as a
+# TfLiteTensor.
+node {
+  calculator: "TfLiteConverterCalculator"
+  input_stream: "IMAGE:transformed_input_video_cpu"
+  output_stream: "TENSORS:image_tensor"
+}
+
+# Runs a TensorFlow Lite model on CPU that takes an image tensor and outputs a
+# vector of tensors representing, for instance, detection boxes/keypoints and
+# scores.
+node {
+  calculator: "TfLiteInferenceCalculator"
+  input_stream: "TENSORS:image_tensor"
+  output_stream: "TENSORS:detection_tensors"
+  options: {
+    [mediapipe.TfLiteInferenceCalculatorOptions.ext] {
+      model_path: "mediapipe/models/face_detection_front.tflite"
+    }
+  }
+}
+
+# Generates a single side packet containing a vector of SSD anchors based on
+# the specification in the options.
+node {
+  calculator: "SsdAnchorsCalculator"
+  output_side_packet: "anchors"
+  options: {
+    [mediapipe.SsdAnchorsCalculatorOptions.ext] {
+      num_layers: 4
+      min_scale: 0.1484375
+      max_scale: 0.75
+      input_size_height: 128
+      input_size_width: 128
+      anchor_offset_x: 0.5
+      anchor_offset_y: 0.5
+      strides: 8
+      strides: 16
+      strides: 16
+      strides: 16
+      aspect_ratios: 1.0
+      fixed_anchor_size: true
+    }
+  }
+}
+
+# Decodes the detection tensors generated by the TensorFlow Lite model, based on
+# the SSD anchors and the specification in the options, into a vector of
+# detections. Each detection describes a detected object.
+node {
+  calculator: "TfLiteTensorsToDetectionsCalculator"
+  input_stream: "TENSORS:detection_tensors"
+  input_side_packet: "ANCHORS:anchors"
+  output_stream: "DETECTIONS:detections"
+  options: {
+    [mediapipe.TfLiteTensorsToDetectionsCalculatorOptions.ext] {
+      num_classes: 1
+      num_boxes: 896
+      num_coords: 16
+      box_coord_offset: 0
+      keypoint_coord_offset: 4
+      num_keypoints: 6
+      num_values_per_keypoint: 2
+      sigmoid_score: true
+      score_clipping_thresh: 100.0
+      reverse_output_order: true
+      x_scale: 128.0
+      y_scale: 128.0
+      h_scale: 128.0
+      w_scale: 128.0
+      min_score_thresh: 0.6
+    }
+  }
+}
+
+# Performs non-max suppression to remove excessive detections.
+node {
+  calculator: "NonMaxSuppressionCalculator"
+  input_stream: "detections"
+  output_stream: "filtered_detections"
+  options: {
+    [mediapipe.NonMaxSuppressionCalculatorOptions.ext] {
+      min_suppression_threshold: 0.3
+      overlap_type: INTERSECTION_OVER_UNION
+      algorithm: WEIGHTED
+      return_empty_detections: true
+    }
+  }
+}
+
+# Maps detection label IDs to the corresponding label text ("Face"). The label
+# map is provided in the label_map_path option.
+node {
+  calculator: "DetectionLabelIdToTextCalculator"
+  input_stream: "filtered_detections"
+  output_stream: "labeled_detections"
+  options: {
+    [mediapipe.DetectionLabelIdToTextCalculatorOptions.ext] {
+      label_map_path: "mediapipe/models/face_detection_front_labelmap.txt"
+    }
+  }
+}
+
+# Adjusts detection locations (already normalized to [0.f, 1.f]) on the
+# letterboxed image (after image transformation with the FIT scale mode) to the
+# corresponding locations on the same image with the letterbox removed (the
+# input image to the graph before image transformation).
+node {
+  calculator: "DetectionLetterboxRemovalCalculator"
+  input_stream: "DETECTIONS:labeled_detections"
+  input_stream: "LETTERBOX_PADDING:letterbox_padding"
+  output_stream: "DETECTIONS:output_detections"
+}

mediapipe/examples/desktop/demo_run_graph_main.cc

@@ -13,6 +13,7 @@
 // limitations under the License.
 //
 // An example of sending OpenCV webcam frames into a MediaPipe graph.
+#include <cstdlib>
 
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/formats/image_frame.h"
@@ -65,16 +66,7 @@ DEFINE_string(output_video_path, "",
 
   cv::VideoWriter writer;
   const bool save_video = !FLAGS_output_video_path.empty();
-  if (save_video) {
-    LOG(INFO) << "Prepare video writer.";
-    cv::Mat test_frame;
-    capture.read(test_frame);                    // Consume first frame.
-    capture.set(cv::CAP_PROP_POS_AVI_RATIO, 0);  // Rewind to beginning.
-    writer.open(FLAGS_output_video_path,
-                mediapipe::fourcc('a', 'v', 'c', '1'),  // .mp4
-                capture.get(cv::CAP_PROP_FPS), test_frame.size());
-    RET_CHECK(writer.isOpened());
-  } else {
+  if (!save_video) {
     cv::namedWindow(kWindowName, /*flags=WINDOW_AUTOSIZE*/ 1);
 #if (CV_MAJOR_VERSION >= 3) && (CV_MINOR_VERSION >= 2)
     capture.set(cv::CAP_PROP_FRAME_WIDTH, 640);
@@ -89,7 +81,6 @@ DEFINE_string(output_video_path, "",
   MP_RETURN_IF_ERROR(graph.StartRun({}));
 
   LOG(INFO) << "Start grabbing and processing frames.";
-  size_t frame_timestamp = 0;
   bool grab_frames = true;
   while (grab_frames) {
     // Capture opencv camera or video frame.
@@ -110,9 +101,11 @@ DEFINE_string(output_video_path, "",
     camera_frame.copyTo(input_frame_mat);
 
     // Send image packet into the graph.
+    size_t frame_timestamp_us =
+        (double)cv::getTickCount() / (double)cv::getTickFrequency() * 1e6;
     MP_RETURN_IF_ERROR(graph.AddPacketToInputStream(
         kInputStream, mediapipe::Adopt(input_frame.release())
-                          .At(mediapipe::Timestamp(frame_timestamp++))));
+                          .At(mediapipe::Timestamp(frame_timestamp_us))));
 
     // Get the graph result packet, or stop if that fails.
     mediapipe::Packet packet;
@@ -123,6 +116,13 @@ DEFINE_string(output_video_path, "",
     cv::Mat output_frame_mat = mediapipe::formats::MatView(&output_frame);
     cv::cvtColor(output_frame_mat, output_frame_mat, cv::COLOR_RGB2BGR);
     if (save_video) {
+      if (!writer.isOpened()) {
+        LOG(INFO) << "Prepare video writer.";
+        writer.open(FLAGS_output_video_path,
+                    mediapipe::fourcc('a', 'v', 'c', '1'),  // .mp4
+                    capture.get(cv::CAP_PROP_FPS), output_frame_mat.size());
+        RET_CHECK(writer.isOpened());
+      }
       writer.write(output_frame_mat);
     } else {
       cv::imshow(kWindowName, output_frame_mat);
@@ -144,8 +144,9 @@ int main(int argc, char** argv) {
   ::mediapipe::Status run_status = RunMPPGraph();
   if (!run_status.ok()) {
     LOG(ERROR) << "Failed to run the graph: " << run_status.message();
+    return EXIT_FAILURE;
   } else {
     LOG(INFO) << "Success!";
   }
-  return 0;
+  return EXIT_SUCCESS;
 }

mediapipe/examples/desktop/demo_run_graph_main_gpu.cc

@@ -14,6 +14,7 @@
 //
 // An example of sending OpenCV webcam frames into a MediaPipe graph.
 // This example requires a linux computer and a GPU with EGL support drivers.
+#include <cstdlib>
 
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/formats/image_frame.h"
@@ -75,16 +76,7 @@ DEFINE_string(output_video_path, "",
 
   cv::VideoWriter writer;
   const bool save_video = !FLAGS_output_video_path.empty();
-  if (save_video) {
-    LOG(INFO) << "Prepare video writer.";
-    cv::Mat test_frame;
-    capture.read(test_frame);                    // Consume first frame.
-    capture.set(cv::CAP_PROP_POS_AVI_RATIO, 0);  // Rewind to beginning.
-    writer.open(FLAGS_output_video_path,
-                mediapipe::fourcc('a', 'v', 'c', '1'),  // .mp4
-                capture.get(cv::CAP_PROP_FPS), test_frame.size());
-    RET_CHECK(writer.isOpened());
-  } else {
+  if (!save_video) {
     cv::namedWindow(kWindowName, /*flags=WINDOW_AUTOSIZE*/ 1);
 #if (CV_MAJOR_VERSION >= 3) && (CV_MINOR_VERSION >= 2)
     capture.set(cv::CAP_PROP_FRAME_WIDTH, 640);
@@ -99,7 +91,6 @@ DEFINE_string(output_video_path, "",
   MP_RETURN_IF_ERROR(graph.StartRun({}));
 
   LOG(INFO) << "Start grabbing and processing frames.";
-  size_t frame_timestamp = 0;
   bool grab_frames = true;
   while (grab_frames) {
     // Capture opencv camera or video frame.
@@ -120,8 +111,10 @@ DEFINE_string(output_video_path, "",
     camera_frame.copyTo(input_frame_mat);
 
     // Prepare and add graph input packet.
+    size_t frame_timestamp_us =
+        (double)cv::getTickCount() / (double)cv::getTickFrequency() * 1e6;
     MP_RETURN_IF_ERROR(
-        gpu_helper.RunInGlContext([&input_frame, &frame_timestamp, &graph,
+        gpu_helper.RunInGlContext([&input_frame, &frame_timestamp_us, &graph,
                                    &gpu_helper]() -> ::mediapipe::Status {
           // Convert ImageFrame to GpuBuffer.
           auto texture = gpu_helper.CreateSourceTexture(*input_frame.get());
@@ -131,7 +124,7 @@ DEFINE_string(output_video_path, "",
           // Send GPU image packet into the graph.
           MP_RETURN_IF_ERROR(graph.AddPacketToInputStream(
               kInputStream, mediapipe::Adopt(gpu_frame.release())
-                                .At(mediapipe::Timestamp(frame_timestamp++))));
+                                .At(mediapipe::Timestamp(frame_timestamp_us))));
           return ::mediapipe::OkStatus();
         }));
 
@@ -163,6 +156,13 @@ DEFINE_string(output_video_path, "",
     cv::Mat output_frame_mat = mediapipe::formats::MatView(output_frame.get());
     cv::cvtColor(output_frame_mat, output_frame_mat, cv::COLOR_RGB2BGR);
     if (save_video) {
+      if (!writer.isOpened()) {
+        LOG(INFO) << "Prepare video writer.";
+        writer.open(FLAGS_output_video_path,
+                    mediapipe::fourcc('a', 'v', 'c', '1'),  // .mp4
+                    capture.get(cv::CAP_PROP_FPS), output_frame_mat.size());
+        RET_CHECK(writer.isOpened());
+      }
       writer.write(output_frame_mat);
     } else {
       cv::imshow(kWindowName, output_frame_mat);
@@ -184,8 +184,9 @@ int main(int argc, char** argv) {
   ::mediapipe::Status run_status = RunMPPGraph();
   if (!run_status.ok()) {
     LOG(ERROR) << "Failed to run the graph: " << run_status.message();
+    return EXIT_FAILURE;
   } else {
     LOG(INFO) << "Success!";
   }
-  return 0;
+  return EXIT_SUCCESS;
 }

mediapipe/examples/desktop/media_sequence/kinetics_dataset.py

@@ -56,6 +56,9 @@ with the following lines:
 This data is structured for per-clip action classification where images is
 the sequence of images and labels are a one-hot encoded value. See
 as_dataset() for more details.
+
+Note that the number of videos changes in the data set over time, so it will
+likely be necessary to change the expected number of examples.
 """
 
 from __future__ import absolute_import
@@ -93,15 +96,15 @@ FILEPATTERN = "kinetics_700_%s_25fps_rgb_flow"
 SPLITS = {
     "train": {
         "shards": 1000,
-        "examples": 541279
+        "examples": 540247
     },
     "validate": {
         "shards": 100,
-        "examples": 34688
+        "examples": 34610
     },
     "test": {
         "shards": 100,
-        "examples": 69278
+        "examples": 69103
     },
     "custom": {
         "csv": None,  # Add a CSV for your own data here.
@@ -121,7 +124,8 @@ class Kinetics(object):
     self.path_to_data = path_to_data
 
   def as_dataset(self, split, shuffle=False, repeat=False,
-                 serialized_prefetch_size=32, decoded_prefetch_size=32):
+                 serialized_prefetch_size=32, decoded_prefetch_size=32,
+                 parse_labels=True):
     """Returns Kinetics as a tf.data.Dataset.
 
     After running this function, calling padded_batch() on the Dataset object
@@ -135,20 +139,29 @@ class Kinetics(object):
       repeat: if true, repeats the data set forever.
       serialized_prefetch_size: the buffer size for reading from disk.
       decoded_prefetch_size: the buffer size after decoding.
+      parse_labels: if true, also returns the "labels" below. The only
+        case where this should be false is if the data set was not constructed
+        with a label map, resulting in this field being missing.
     Returns:
       A tf.data.Dataset object with the following structure: {
         "images": float tensor, shape [time, height, width, channels]
         "flow": float tensor, shape [time, height, width, 2]
-        "labels": float32 tensor, shape [num_classes], one hot encoded
         "num_frames": int32 tensor, shape [], number of frames in the sequence
+        "labels": float32 tensor, shape [num_classes], one hot encoded. Only
+          present if parse_labels is true.
     """
+    logging.info("If you see an error about labels, and you don't supply "
+                 "labels in your CSV, set parse_labels=False")
     def parse_fn(sequence_example):
       """Parses a Kinetics example."""
       context_features = {
           ms.get_example_id_key(): ms.get_example_id_default_parser(),
-          ms.get_clip_label_string_key(): tf.FixedLenFeature((), tf.string),
-          ms.get_clip_label_index_key(): tf.FixedLenFeature((), tf.int64),
       }
+      if parse_labels:
+        context_features[
+            ms.get_clip_label_string_key()] = tf.FixedLenFeature((), tf.string)
+        context_features[
+            ms.get_clip_label_index_key()] = tf.FixedLenFeature((), tf.int64)
 
       sequence_features = {
           ms.get_image_encoded_key(): ms.get_image_encoded_default_parser(),
@@ -158,8 +171,6 @@ class Kinetics(object):
       parsed_context, parsed_sequence = tf.io.parse_single_sequence_example(
           sequence_example, context_features, sequence_features)
 
-      target = tf.one_hot(parsed_context[ms.get_clip_label_index_key()], 700)
-
       images = tf.image.convert_image_dtype(
           tf.map_fn(tf.image.decode_jpeg,
                     parsed_sequence[ms.get_image_encoded_key()],
@@ -177,11 +188,13 @@ class Kinetics(object):
       flow = (flow[:, :, :, :2] - 0.5) * 2 * 20.
 
       output_dict = {
-          "labels": target,
           "images": images,
           "flow": flow,
           "num_frames": num_frames,
       }
+      if parse_labels:
+        target = tf.one_hot(parsed_context[ms.get_clip_label_index_key()], 700)
+        output_dict["labels"] = target
       return output_dict
 
     if split not in SPLITS:

mediapipe/examples/desktop/media_sequence/run_graph_file_io_main.cc

@@ -15,6 +15,7 @@
 // A simple main function to run a MediaPipe graph. Input side packets are read
 // from files provided via the command line and output side packets are written
 // to disk.
+#include <cstdlib>
 
 #include "absl/strings/str_split.h"
 #include "mediapipe/framework/calculator_framework.h"
@@ -87,8 +88,9 @@ int main(int argc, char** argv) {
   ::mediapipe::Status run_status = RunMPPGraph();
   if (!run_status.ok()) {
     LOG(ERROR) << "Failed to run the graph: " << run_status.message();
+    return EXIT_FAILURE;
   } else {
     LOG(INFO) << "Success!";
   }
-  return 0;
+  return EXIT_SUCCESS;
 }

mediapipe/examples/desktop/object_tracking/BUILD

@@ -0,0 +1,25 @@
+# Copyright 2019 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+licenses(["notice"])  # Apache 2.0
+
+package(default_visibility = ["//mediapipe/examples:__subpackages__"])
+
+cc_binary(
+    name = "object_tracking_cpu",
+    deps = [
+        "//mediapipe/examples/desktop:demo_run_graph_main",
+        "//mediapipe/graphs/tracking:desktop_calculators",
+    ],
+)

mediapipe/examples/desktop/simple_run_graph_main.cc

@@ -13,6 +13,7 @@
 // limitations under the License.
 //
 // A simple main function to run a MediaPipe graph.
+#include <cstdlib>
 #include <fstream>
 #include <iostream>
 #include <map>
@@ -143,8 +144,9 @@ int main(int argc, char** argv) {
   ::mediapipe::Status run_status = RunMPPGraph();
   if (!run_status.ok()) {
     LOG(ERROR) << "Failed to run the graph: " << run_status.message();
+    return EXIT_FAILURE;
   } else {
     LOG(INFO) << "Success!";
   }
-  return 0;
+  return EXIT_SUCCESS;
 }

mediapipe/examples/desktop/youtube8m/README.md

@@ -1,6 +1,8 @@
 ### Steps to run the YouTube-8M feature extraction graph
 
-1.  Checkout the mediapipe repository.
+1.  Checkout the repository and follow
+    [the installation instructions](https://github.com/google/mediapipe/blob/master/mediapipe/docs/install.md)
+    to set up MediaPipe.
 
     ```bash
     git clone https://github.com/google/mediapipe.git

mediapipe/examples/desktop/youtube8m/extract_yt8m_features.cc

@@ -15,6 +15,7 @@
 // A simple main function to run a MediaPipe graph. Input side packets are read
 // from files provided via the command line and output side packets are written
 // to disk.
+#include <cstdlib>
 
 #include "absl/strings/str_split.h"
 #include "mediapipe/framework/calculator_framework.h"
@@ -128,8 +129,9 @@ int main(int argc, char** argv) {
   ::mediapipe::Status run_status = RunMPPGraph();
   if (!run_status.ok()) {
     LOG(ERROR) << "Failed to run the graph: " << run_status.message();
+    return EXIT_FAILURE;
   } else {
     LOG(INFO) << "Success!";
   }
-  return 0;
+  return EXIT_SUCCESS;
 }

mediapipe/examples/desktop/youtube8m/generate_input_sequence_example.py

@@ -21,7 +21,7 @@ import sys
 
 from absl import app
 from absl import flags
-import tensorflow as tf
+import tensorflow.compat.v1 as tf
 from mediapipe.util.sequence import media_sequence as ms
 
 FLAGS = flags.FLAGS

mediapipe/examples/desktop/youtube8m/generate_vggish_frozen_graph.py

@@ -24,7 +24,7 @@ import os
 import sys
 
 from absl import app
-import tensorflow as tf
+import tensorflow.compat.v1 as tf
 from tensorflow.python.tools import freeze_graph
 
 BASE_DIR = '/tmp/mediapipe/'