Project import generated by Copybara.

GitOrigin-RevId: 0517756260533d374df93679965ca662d0ec6943

mediapipe/examples/desktop/autoflip/BUILD

@@ -0,0 +1,56 @@
+load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
+
+# Copyright 2019 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+licenses(["notice"])  # Apache 2.0
+
+package(default_visibility = ["//mediapipe/examples:__subpackages__"])
+
+proto_library(
+    name = "autoflip_messages_proto",
+    srcs = ["autoflip_messages.proto"],
+    deps = [
+        "//mediapipe/framework:calculator_proto",
+    ],
+)
+
+mediapipe_cc_proto_library(
+    name = "autoflip_messages_cc_proto",
+    srcs = ["autoflip_messages.proto"],
+    cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
+    visibility = ["//mediapipe/examples:__subpackages__"],
+    deps = [":autoflip_messages_proto"],
+)
+
+cc_binary(
+    name = "run_autoflip",
+    deps = [
+        "//mediapipe/calculators/core:packet_thinner_calculator",
+        "//mediapipe/calculators/image:scale_image_calculator",
+        "//mediapipe/calculators/video:opencv_video_decoder_calculator",
+        "//mediapipe/calculators/video:opencv_video_encoder_calculator",
+        "//mediapipe/calculators/video:video_pre_stream_calculator",
+        "//mediapipe/examples/desktop:simple_run_graph_main",
+        "//mediapipe/examples/desktop/autoflip/calculators:border_detection_calculator",
+        "//mediapipe/examples/desktop/autoflip/calculators:face_to_region_calculator",
+        "//mediapipe/examples/desktop/autoflip/calculators:localization_to_region_calculator",
+        "//mediapipe/examples/desktop/autoflip/calculators:scene_cropping_calculator",
+        "//mediapipe/examples/desktop/autoflip/calculators:shot_boundary_calculator",
+        "//mediapipe/examples/desktop/autoflip/calculators:signal_fusing_calculator",
+        "//mediapipe/examples/desktop/autoflip/calculators:video_filtering_calculator",
+        "//mediapipe/examples/desktop/autoflip/subgraph:autoflip_face_detection_subgraph",
+        "//mediapipe/examples/desktop/autoflip/subgraph:autoflip_object_detection_subgraph",
+    ],
+)

mediapipe/examples/desktop/autoflip/README.md

@@ -0,0 +1,25 @@
+### Steps to run the AutoFlip video cropping graph
+
+1.  Checkout the repository and follow
+    [the installation instructions](https://github.com/google/mediapipe/blob/master/mediapipe/docs/install.md)
+    to set up MediaPipe.
+
+    ```bash
+    git clone https://github.com/google/mediapipe.git
+    cd mediapipe
+    ```
+
+2.  Build and run the run_autoflip binary to process a local video.
+
+    ```bash
+    bazel build -c opt --define MEDIAPIPE_DISABLE_GPU=1 \
+      mediapipe/examples/desktop/autoflip:run_autoflip
+
+    GLOG_logtostderr=1 bazel-bin/mediapipe/examples/desktop/autoflip/run_autoflip \
+      --calculator_graph_config_file=mediapipe/examples/desktop/autoflip/autoflip_graph.pbtxt \
+      --input_side_packets=input_video_path=/absolute/path/to/the/local/video/file,\
+        output_video_path=/absolute/path/to/save/the/output/video/file,\
+        aspect_ratio=width:height
+    ```
+
+3.  View the cropped video.

mediapipe/examples/desktop/autoflip/autoflip_graph.pbtxt

@@ -0,0 +1,202 @@
+# Autoflip graph that only renders the final cropped video. For use with
+# end user applications.
+max_queue_size: -1
+
+# VIDEO_PREP: Decodes an input video file into images and a video header.
+node {
+  calculator: "OpenCvVideoDecoderCalculator"
+  input_side_packet: "INPUT_FILE_PATH:input_video_path"
+  output_stream: "VIDEO:video_raw"
+  output_stream: "VIDEO_PRESTREAM:video_header"
+  output_side_packet: "SAVED_AUDIO_PATH:audio_path"
+}
+
+# VIDEO_PREP: Scale the input video before feature extraction.
+node {
+  calculator: "ScaleImageCalculator"
+  input_stream: "FRAMES:video_raw"
+  input_stream: "VIDEO_HEADER:video_header"
+  output_stream: "FRAMES:video_frames_scaled"
+  options: {
+    [mediapipe.ScaleImageCalculatorOptions.ext]: {
+      preserve_aspect_ratio: true
+      output_format: SRGB
+      target_width: 480
+      algorithm: DEFAULT_WITHOUT_UPSCALE
+    }
+  }
+}
+
+# VIDEO_PREP: Create a low frame rate stream for feature extraction.
+node {
+  calculator: "PacketThinnerCalculator"
+  input_stream: "video_frames_scaled"
+  output_stream: "video_frames_scaled_downsampled"
+  options: {
+    [mediapipe.PacketThinnerCalculatorOptions.ext]: {
+      thinner_type: ASYNC
+      period: 500000
+    }
+  }
+}
+
+# DETECTION: find borders around the video and major background color.
+node {
+  calculator: "BorderDetectionCalculator"
+  input_stream: "VIDEO:video_raw"
+  output_stream: "DETECTED_BORDERS:borders"
+}
+
+# DETECTION: find shot/scene boundaries on the full frame rate stream.
+node {
+  calculator: "ShotBoundaryCalculator"
+  input_stream: "VIDEO:video_frames_scaled"
+  output_stream: "IS_SHOT_CHANGE:shot_change"
+  options {
+    [mediapipe.autoflip.ShotBoundaryCalculatorOptions.ext] {
+      min_shot_span: 0.2
+      min_motion: 0.3
+      window_size: 15
+      min_shot_measure: 10
+      min_motion_with_shot_measure: 0.05
+    }
+  }
+}
+
+# DETECTION: find faces on the down sampled stream
+node {
+  calculator: "AutoFlipFaceDetectionSubgraph"
+  input_stream: "VIDEO:video_frames_scaled_downsampled"
+  output_stream: "DETECTIONS:face_detections"
+}
+node {
+  calculator: "FaceToRegionCalculator"
+  input_stream: "VIDEO:video_frames_scaled_downsampled"
+  input_stream: "FACES:face_detections"
+  output_stream: "REGIONS:face_regions"
+}
+
+# DETECTION: find objects on the down sampled stream
+node {
+  calculator: "AutoFlipObjectDetectionSubgraph"
+  input_stream: "VIDEO:video_frames_scaled_downsampled"
+  output_stream: "DETECTIONS:object_detections"
+}
+node {
+  calculator: "LocalizationToRegionCalculator"
+  input_stream: "DETECTIONS:object_detections"
+  output_stream: "REGIONS:object_regions"
+  options {
+    [mediapipe.autoflip.LocalizationToRegionCalculatorOptions.ext] {
+      output_all_signals: true
+    }
+  }
+}
+
+# SIGNAL FUSION: Combine detections (with weights) on each frame
+node {
+  calculator: "SignalFusingCalculator"
+  input_stream: "shot_change"
+  input_stream: "face_regions"
+  input_stream: "object_regions"
+  output_stream: "salient_regions"
+  options {
+    [mediapipe.autoflip.SignalFusingCalculatorOptions.ext] {
+      signal_settings {
+        type { standard: FACE_CORE_LANDMARKS }
+        min_score: 0.85
+        max_score: 0.9
+        is_required: false
+      }
+      signal_settings {
+        type { standard: FACE_ALL_LANDMARKS }
+        min_score: 0.8
+        max_score: 0.85
+        is_required: false
+      }
+      signal_settings {
+        type { standard: FACE_FULL }
+        min_score: 0.8
+        max_score: 0.85
+        is_required: false
+      }
+      signal_settings {
+        type: { standard: HUMAN }
+        min_score: 0.75
+        max_score: 0.8
+        is_required: false
+      }
+      signal_settings {
+        type: { standard: PET }
+        min_score: 0.7
+        max_score: 0.75
+        is_required: false
+      }
+      signal_settings {
+        type: { standard: CAR }
+        min_score: 0.7
+        max_score: 0.75
+        is_required: false
+      }
+      signal_settings {
+        type: { standard: OBJECT }
+        min_score: 0.1
+        max_score: 0.2
+        is_required: false
+      }
+    }
+  }
+}
+
+# CROPPING: make decisions about how to crop each frame.
+node {
+  calculator: "SceneCroppingCalculator"
+  input_side_packet: "EXTERNAL_ASPECT_RATIO:aspect_ratio"
+  input_stream: "VIDEO_FRAMES:video_raw"
+  input_stream: "KEY_FRAMES:video_frames_scaled_downsampled"
+  input_stream: "DETECTION_FEATURES:salient_regions"
+  input_stream: "STATIC_FEATURES:borders"
+  input_stream: "SHOT_BOUNDARIES:shot_change"
+  output_stream: "CROPPED_FRAMES:cropped_frames"
+  options: {
+    [mediapipe.autoflip.SceneCroppingCalculatorOptions.ext]: {
+      max_scene_size: 600
+      key_frame_crop_options: {
+        score_aggregation_type: CONSTANT
+      }
+      scene_camera_motion_analyzer_options: {
+        motion_stabilization_threshold_percent: 0.3
+        salient_point_bound: 0.499
+      }
+      padding_parameters: {
+        blur_cv_size: 200
+        overlay_opacity: 0.6
+      }
+      target_size_type: MAXIMIZE_TARGET_DIMENSION
+    }
+  }
+}
+
+# ENCODING(required): encode the video stream for the final cropped output.
+node {
+  calculator: "VideoPreStreamCalculator"
+  # Fetch frame format and dimension from input frames.
+  input_stream: "FRAME:cropped_frames"
+  # Copying frame rate and duration from original video.
+  input_stream: "VIDEO_PRESTREAM:video_header"
+  output_stream: "output_frames_video_header"
+}
+
+node {
+  calculator: "OpenCvVideoEncoderCalculator"
+  input_stream: "VIDEO:cropped_frames"
+  input_stream: "VIDEO_PRESTREAM:output_frames_video_header"
+  input_side_packet: "OUTPUT_FILE_PATH:output_video_path"
+  input_side_packet: "AUDIO_FILE_PATH:audio_path"
+  options: {
+    [mediapipe.OpenCvVideoEncoderCalculatorOptions.ext]: {
+      codec: "avc1"
+      video_format: "mp4"
+    }
+  }
+}

mediapipe/examples/desktop/autoflip/autoflip_graph_development.pbtxt

@@ -0,0 +1,252 @@
+# Autoflip graph that renders the final cropped video and debugging videos.
+# For use by developers who may be adding signals and adjusting weights.
+max_queue_size: -1
+
+# VIDEO_PREP: Decodes an input video file into images and a video header.
+node {
+  calculator: "OpenCvVideoDecoderCalculator"
+  input_side_packet: "INPUT_FILE_PATH:input_video_path"
+  output_stream: "VIDEO:video_raw"
+  output_stream: "VIDEO_PRESTREAM:video_header"
+  output_side_packet: "SAVED_AUDIO_PATH:audio_path"
+}
+
+# VIDEO_PREP: Scale the input video before feature extraction.
+node {
+  calculator: "ScaleImageCalculator"
+  input_stream: "FRAMES:video_raw"
+  input_stream: "VIDEO_HEADER:video_header"
+  output_stream: "FRAMES:video_frames_scaled"
+  options: {
+    [mediapipe.ScaleImageCalculatorOptions.ext]: {
+      preserve_aspect_ratio: true
+      output_format: SRGB
+      target_width: 480
+      algorithm: DEFAULT_WITHOUT_UPSCALE
+    }
+  }
+}
+
+# VIDEO_PREP: Create a low frame rate stream for feature extraction.
+node {
+  calculator: "PacketThinnerCalculator"
+  input_stream: "video_frames_scaled"
+  output_stream: "video_frames_scaled_downsampled"
+  options: {
+    [mediapipe.PacketThinnerCalculatorOptions.ext]: {
+      thinner_type: ASYNC
+      period: 500000
+    }
+  }
+}
+
+# DETECTION: find borders around the video and major background color.
+node {
+  calculator: "BorderDetectionCalculator"
+  input_stream: "VIDEO:video_raw"
+  output_stream: "DETECTED_BORDERS:borders"
+}
+
+# DETECTION: find shot/scene boundaries on the full frame rate stream.
+node {
+  calculator: "ShotBoundaryCalculator"
+  input_stream: "VIDEO:video_frames_scaled"
+  output_stream: "IS_SHOT_CHANGE:shot_change"
+  options {
+    [mediapipe.autoflip.ShotBoundaryCalculatorOptions.ext] {
+      min_shot_span: 0.2
+      min_motion: 0.3
+      window_size: 15
+      min_shot_measure: 10
+      min_motion_with_shot_measure: 0.05
+    }
+  }
+}
+
+# DETECTION: find faces on the down sampled stream
+node {
+  calculator: "AutoFlipFaceDetectionSubgraph"
+  input_stream: "VIDEO:video_frames_scaled_downsampled"
+  output_stream: "DETECTIONS:face_detections"
+}
+node {
+  calculator: "FaceToRegionCalculator"
+  input_stream: "VIDEO:video_frames_scaled_downsampled"
+  input_stream: "FACES:face_detections"
+  output_stream: "REGIONS:face_regions"
+}
+
+# DETECTION: find objects on the down sampled stream
+node {
+  calculator: "AutoFlipObjectDetectionSubgraph"
+  input_stream: "VIDEO:video_frames_scaled_downsampled"
+  output_stream: "DETECTIONS:object_detections"
+}
+node {
+  calculator: "LocalizationToRegionCalculator"
+  input_stream: "DETECTIONS:object_detections"
+  output_stream: "REGIONS:object_regions"
+  options {
+    [mediapipe.autoflip.LocalizationToRegionCalculatorOptions.ext] {
+      output_all_signals: true
+    }
+  }
+}
+
+# SIGNAL FUSION: Combine detections (with weights) on each frame
+node {
+  calculator: "SignalFusingCalculator"
+  input_stream: "shot_change"
+  input_stream: "face_regions"
+  input_stream: "object_regions"
+  output_stream: "salient_regions"
+  options {
+    [mediapipe.autoflip.SignalFusingCalculatorOptions.ext] {
+      signal_settings {
+        type { standard: FACE_CORE_LANDMARKS }
+        min_score: 0.85
+        max_score: 0.9
+        is_required: false
+      }
+      signal_settings {
+        type { standard: FACE_ALL_LANDMARKS }
+        min_score: 0.8
+        max_score: 0.85
+        is_required: false
+      }
+      signal_settings {
+        type { standard: FACE_FULL }
+        min_score: 0.8
+        max_score: 0.85
+        is_required: false
+      }
+      signal_settings {
+        type: { standard: HUMAN }
+        min_score: 0.75
+        max_score: 0.8
+        is_required: false
+      }
+      signal_settings {
+        type: { standard: PET }
+        min_score: 0.7
+        max_score: 0.75
+        is_required: false
+      }
+      signal_settings {
+        type: { standard: CAR }
+        min_score: 0.7
+        max_score: 0.75
+        is_required: false
+      }
+      signal_settings {
+        type: { standard: OBJECT }
+        min_score: 0.1
+        max_score: 0.2
+        is_required: false
+      }
+    }
+  }
+}
+
+# CROPPING: make decisions about how to crop each frame.
+node {
+  calculator: "SceneCroppingCalculator"
+  input_side_packet: "EXTERNAL_ASPECT_RATIO:aspect_ratio"
+  input_stream: "VIDEO_FRAMES:video_raw"
+  input_stream: "KEY_FRAMES:video_frames_scaled_downsampled"
+  input_stream: "DETECTION_FEATURES:salient_regions"
+  input_stream: "STATIC_FEATURES:borders"
+  input_stream: "SHOT_BOUNDARIES:shot_change"
+  output_stream: "CROPPED_FRAMES:cropped_frames"
+  output_stream: "KEY_FRAME_CROP_REGION_VIZ_FRAMES:key_frame_crop_viz_frames"
+  output_stream: "SALIENT_POINT_FRAME_VIZ_FRAMES:salient_point_viz_frames"
+  options: {
+    [mediapipe.autoflip.SceneCroppingCalculatorOptions.ext]: {
+      max_scene_size: 600
+      key_frame_crop_options: {
+        score_aggregation_type: CONSTANT
+      }
+      scene_camera_motion_analyzer_options: {
+        motion_stabilization_threshold_percent: 0.3
+        salient_point_bound: 0.499
+      }
+      padding_parameters: {
+        blur_cv_size: 200
+        overlay_opacity: 0.6
+      }
+      target_size_type: MAXIMIZE_TARGET_DIMENSION
+    }
+  }
+}
+
+# ENCODING(required): encode the video stream for the final cropped output.
+node {
+  calculator: "VideoPreStreamCalculator"
+  # Fetch frame format and dimension from input frames.
+  input_stream: "FRAME:cropped_frames"
+  # Copying frame rate and duration from original video.
+  input_stream: "VIDEO_PRESTREAM:video_header"
+  output_stream: "output_frames_video_header"
+}
+
+node {
+  calculator: "OpenCvVideoEncoderCalculator"
+  input_stream: "VIDEO:cropped_frames"
+  input_stream: "VIDEO_PRESTREAM:output_frames_video_header"
+  input_side_packet: "OUTPUT_FILE_PATH:output_video_path"
+  input_side_packet: "AUDIO_FILE_PATH:audio_path"
+  options: {
+    [mediapipe.OpenCvVideoEncoderCalculatorOptions.ext]: {
+      codec: "avc1"
+      video_format: "mp4"
+    }
+  }
+}
+
+# ENCODING(optional): encode the video stream for the key_frame_crop_viz_frames
+# output.  Draws boxes around required and non-required objects.
+node {
+  calculator: "VideoPreStreamCalculator"
+  # Fetch frame format and dimension from input frames.
+  input_stream: "FRAME:key_frame_crop_viz_frames"
+  # Copying frame rate and duration from original video.
+  input_stream: "VIDEO_PRESTREAM:video_header"
+  output_stream: "key_frame_crop_viz_frames_header"
+}
+
+node {
+  calculator: "OpenCvVideoEncoderCalculator"
+  input_stream: "VIDEO:key_frame_crop_viz_frames"
+  input_stream: "VIDEO_PRESTREAM:key_frame_crop_viz_frames_header"
+  input_side_packet: "OUTPUT_FILE_PATH:key_frame_crop_viz_frames_path"
+  options: {
+    [mediapipe.OpenCvVideoEncoderCalculatorOptions.ext]: {
+      codec: "avc1"
+      video_format: "mp4"
+    }
+  }
+}
+
+# ENCODING(optional): encode the video stream for the salient_point_viz_frames
+# output. Draws the focus points and the scene crop window (red).
+node {
+  calculator: "VideoPreStreamCalculator"
+  # Fetch frame format and dimension from input frames.
+  input_stream: "FRAME:salient_point_viz_frames"
+  # Copying frame rate and duration from original video.
+  input_stream: "VIDEO_PRESTREAM:video_header"
+  output_stream: "salient_point_viz_frames_header"
+}
+
+node {
+  calculator: "OpenCvVideoEncoderCalculator"
+  input_stream: "VIDEO:salient_point_viz_frames"
+  input_stream: "VIDEO_PRESTREAM:salient_point_viz_frames_header"
+  input_side_packet: "OUTPUT_FILE_PATH:salient_point_viz_frames_path"
+  options: {
+    [mediapipe.OpenCvVideoEncoderCalculatorOptions.ext]: {
+      codec: "avc1"
+      video_format: "mp4"
+    }
+  }
+}

mediapipe/examples/desktop/autoflip/autoflip_messages.proto

@@ -0,0 +1,153 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Proto messages used for the AutoFlip Pipeline.
+syntax = "proto2";
+
+package mediapipe.autoflip;
+
+import "mediapipe/framework/calculator.proto";
+
+// Borders detected on the frame as well as non-border color (if present).
+// Next tag: 4
+message StaticFeatures {
+  // A list of the static parts for a frame.
+  repeated Border border = 1;
+  // The background color (only set if solid color).
+  optional Color solid_background = 2;
+  // Area of the image that is not a border.
+  optional Rect non_static_area = 3;
+}
+
+// A static border area within the video.
+// Next tag: 3
+message Border {
+  // Original location within the input frame.
+  optional Rect border_position = 1;
+  // Position for static area.
+  // Next tag: 3
+  enum RelativePosition {
+    TOP = 1;
+    BOTTOM = 2;
+  }
+  // Top or bottom position.
+  optional RelativePosition relative_position = 2;
+}
+
+// Rectangle (opencv format).
+// Next tag: 5
+message Rect {
+  optional int32 x = 1;
+  optional int32 y = 2;
+  optional int32 width = 3;
+  optional int32 height = 4;
+}
+
+// Color (RGB 8bit)
+// Next tag: 4
+message Color {
+  optional int32 r = 1;
+  optional int32 g = 2;
+  optional int32 b = 3;
+}
+
+// Rectangle (opencv format).
+// Next tag: 5
+message RectF {
+  optional float x = 1;
+  optional float y = 2;
+  optional float width = 3;
+  optional float height = 4;
+}
+
+// An image region of interest (eg a detected face or object), accompanied by an
+// importance score.
+// Next tag: 9
+message SalientRegion {
+  reserved 3;
+  // The bounding box for this region in the image.
+  optional Rect location = 1;
+
+  // The bounding box for this region in the image normalized.
+  optional RectF location_normalized = 8;
+
+  // A score indicating the importance of this region.
+  optional float score = 2;
+
+  // A tracking id used to identify this region across video frames. Not always
+  // set.
+  optional int64 tracking_id = 4;
+
+  // If true, this region is required to be present in the final video (eg it
+  // contains text that cannot be cropped).
+  optional bool is_required = 5 [default = false];
+
+  // Type of signal carried in this message.
+  optional SignalType signal_type = 6;
+
+  // If true, object cannot move in the output window (e.g. text would look
+  // strange moving around).
+  optional bool requires_static_location = 7 [default = false];
+}
+
+// Stores the message type, including standard types (face, object) and custom
+// types defined by a string id.
+// Next tag: 3
+message SignalType {
+  enum StandardType {
+    UNSET = 0;
+    // Full face bounding boxed detected.
+    FACE_FULL = 1;
+    // Face landmarks for eyes, nose, chin only.
+    FACE_CORE_LANDMARKS = 2;
+    // All face landmarks (eyes, ears, nose, chin).
+    FACE_ALL_LANDMARKS = 3;
+    // A specific face landmark.
+    FACE_LANDMARK = 4;
+    HUMAN = 5;
+    CAR = 6;
+    PET = 7;
+    OBJECT = 8;
+    MOTION = 9;
+    TEXT = 10;
+    LOGO = 11;
+    USER_HINT = 12;
+  }
+  oneof Signal {
+    StandardType standard = 1;
+    string custom = 2;
+  }
+}
+
+// Features extracted from a image.
+// Next tag: 3
+message DetectionSet {
+  // Mask image showing pixel-wise values at a given location.
+  optional string encoded_mask = 1;
+  // List of rectangle detections.
+  repeated SalientRegion detections = 2;
+}
+
+// General settings needed for multiple calculators.
+message ConversionOptions {
+  extend mediapipe.CalculatorOptions {
+    optional ConversionOptions ext = 284806832;
+  }
+  // Target output width of the conversion.
+  optional int32 target_width = 1;
+  // Target output height of the conversion.
+  optional int32 target_height = 2;
+}
+
+// TODO: Move other autoflip messages into this area.

mediapipe/examples/desktop/autoflip/calculators/BUILD

@@ -0,0 +1,426 @@
+load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
+
+# Copyright 2019 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+licenses(["notice"])  # Apache 2.0
+
+package(default_visibility = ["//mediapipe/examples:__subpackages__"])
+
+cc_library(
+    name = "border_detection_calculator",
+    srcs = ["border_detection_calculator.cc"],
+    deps = [
+        ":border_detection_calculator_cc_proto",
+        "//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:image_frame",
+        "//mediapipe/framework/formats:image_frame_opencv",
+        "//mediapipe/framework/port:opencv_core",
+        "//mediapipe/framework/port:opencv_imgproc",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+    ],
+    alwayslink = 1,
+)
+
+proto_library(
+    name = "border_detection_calculator_proto",
+    srcs = ["border_detection_calculator.proto"],
+    deps = [
+        "//mediapipe/framework:calculator_proto",
+    ],
+)
+
+mediapipe_cc_proto_library(
+    name = "border_detection_calculator_cc_proto",
+    srcs = ["border_detection_calculator.proto"],
+    cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
+    visibility = ["//mediapipe/examples:__subpackages__"],
+    deps = [":border_detection_calculator_proto"],
+)
+
+cc_test(
+    name = "border_detection_calculator_test",
+    srcs = [
+        "border_detection_calculator_test.cc",
+    ],
+    linkstatic = 1,
+    deps = [
+        ":border_detection_calculator",
+        ":border_detection_calculator_cc_proto",
+        "//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:calculator_runner",
+        "//mediapipe/framework/formats:image_frame",
+        "//mediapipe/framework/formats:image_frame_opencv",
+        "//mediapipe/framework/port:benchmark",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:opencv_core",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
+cc_library(
+    name = "video_filtering_calculator",
+    srcs = ["video_filtering_calculator.cc"],
+    copts = ["-fexceptions"],
+    features = ["-use_header_modules"],  # Incompatible with -fexceptions.
+    deps = [
+        ":video_filtering_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:image_frame",
+        "//mediapipe/framework/formats:image_frame_opencv",
+        "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/strings",
+    ],
+    alwayslink = 1,
+)
+
+proto_library(
+    name = "video_filtering_calculator_proto",
+    srcs = ["video_filtering_calculator.proto"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_proto",
+    ],
+)
+
+mediapipe_cc_proto_library(
+    name = "video_filtering_calculator_cc_proto",
+    srcs = ["video_filtering_calculator.proto"],
+    cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
+    visibility = ["//visibility:public"],
+    deps = [":video_filtering_calculator_proto"],
+)
+
+cc_test(
+    name = "video_filtering_calculator_test",
+    srcs = ["video_filtering_calculator_test.cc"],
+    deps = [
+        ":video_filtering_calculator",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:calculator_runner",
+        "//mediapipe/framework/formats:image_frame",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
+proto_library(
+    name = "scene_cropping_calculator_proto",
+    srcs = ["scene_cropping_calculator.proto"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/examples/desktop/autoflip/quality:cropping_proto",
+        "//mediapipe/framework:calculator_proto",
+    ],
+)
+
+mediapipe_cc_proto_library(
+    name = "scene_cropping_calculator_cc_proto",
+    srcs = ["scene_cropping_calculator.proto"],
+    cc_deps = [
+        "//mediapipe/examples/desktop/autoflip/quality:cropping_cc_proto",
+        "//mediapipe/framework:calculator_cc_proto",
+    ],
+    visibility = ["//visibility:public"],
+    deps = [":scene_cropping_calculator_proto"],
+)
+
+cc_library(
+    name = "scene_cropping_calculator",
+    srcs = ["scene_cropping_calculator.cc"],
+    hdrs = ["scene_cropping_calculator.h"],
+    deps = [
+        ":scene_cropping_calculator_cc_proto",
+        "//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto",
+        "//mediapipe/examples/desktop/autoflip/quality:cropping_cc_proto",
+        "//mediapipe/examples/desktop/autoflip/quality:focus_point_cc_proto",
+        "//mediapipe/examples/desktop/autoflip/quality:frame_crop_region_computer",
+        "//mediapipe/examples/desktop/autoflip/quality:padding_effect_generator",
+        "//mediapipe/examples/desktop/autoflip/quality:piecewise_linear_function",
+        "//mediapipe/examples/desktop/autoflip/quality:polynomial_regression_path_solver",
+        "//mediapipe/examples/desktop/autoflip/quality:scene_camera_motion_analyzer",
+        "//mediapipe/examples/desktop/autoflip/quality:scene_cropper",
+        "//mediapipe/examples/desktop/autoflip/quality:scene_cropping_viz",
+        "//mediapipe/examples/desktop/autoflip/quality:utils",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:timestamp",
+        "//mediapipe/framework/formats:image_frame",
+        "//mediapipe/framework/formats:image_frame_opencv",
+        "//mediapipe/framework/port:opencv_core",
+        "//mediapipe/framework/port:opencv_imgproc",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/memory",
+        "@com_google_absl//absl/strings:str_format",
+    ],
+    alwayslink = 1,  # buildozer: disable=alwayslink-with-hdrs
+)
+
+cc_test(
+    name = "scene_cropping_calculator_test",
+    size = "large",
+    timeout = "long",
+    srcs = ["scene_cropping_calculator_test.cc"],
+    deps = [
+        ":scene_cropping_calculator",
+        "//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:calculator_runner",
+        "//mediapipe/framework/formats:image_frame",
+        "//mediapipe/framework/formats:image_frame_opencv",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:opencv_core",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+    ],
+)
+
+cc_library(
+    name = "signal_fusing_calculator",
+    srcs = ["signal_fusing_calculator.cc"],
+    deps = [
+        ":signal_fusing_calculator_cc_proto",
+        "//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:image_frame",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+    ],
+    alwayslink = 1,
+)
+
+proto_library(
+    name = "signal_fusing_calculator_proto",
+    srcs = ["signal_fusing_calculator.proto"],
+    deps = [
+        "//mediapipe/examples/desktop/autoflip:autoflip_messages_proto",
+        "//mediapipe/framework:calculator_proto",
+    ],
+)
+
+mediapipe_cc_proto_library(
+    name = "signal_fusing_calculator_cc_proto",
+    srcs = ["signal_fusing_calculator.proto"],
+    cc_deps = [
+        "//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto",
+        "//mediapipe/framework:calculator_cc_proto",
+    ],
+    visibility = ["//mediapipe/examples:__subpackages__"],
+    deps = [":signal_fusing_calculator_proto"],
+)
+
+cc_test(
+    name = "signal_fusing_calculator_test",
+    srcs = ["signal_fusing_calculator_test.cc"],
+    linkstatic = 1,
+    deps = [
+        ":signal_fusing_calculator",
+        ":signal_fusing_calculator_cc_proto",
+        "//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:calculator_runner",
+        "//mediapipe/framework/formats:image_frame",
+        "//mediapipe/framework/formats:image_frame_opencv",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
+cc_library(
+    name = "shot_boundary_calculator",
+    srcs = ["shot_boundary_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":shot_boundary_calculator_cc_proto",
+        "//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:timestamp",
+        "//mediapipe/framework/formats:image_frame",
+        "//mediapipe/framework/formats:image_frame_opencv",
+        "//mediapipe/framework/port:opencv_imgproc",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+    ],
+    alwayslink = 1,
+)
+
+proto_library(
+    name = "shot_boundary_calculator_proto",
+    srcs = ["shot_boundary_calculator.proto"],
+    deps = [
+        "//mediapipe/examples/desktop/autoflip:autoflip_messages_proto",
+        "//mediapipe/framework:calculator_proto",
+    ],
+)
+
+mediapipe_cc_proto_library(
+    name = "shot_boundary_calculator_cc_proto",
+    srcs = ["shot_boundary_calculator.proto"],
+    cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
+    visibility = ["//mediapipe/examples:__subpackages__"],
+    deps = [":shot_boundary_calculator_proto"],
+)
+
+cc_test(
+    name = "shot_boundary_calculator_test",
+    srcs = ["shot_boundary_calculator_test.cc"],
+    data = ["//mediapipe/examples/desktop/autoflip/calculators/testdata:test_images"],
+    linkstatic = 1,
+    deps = [
+        ":shot_boundary_calculator",
+        ":shot_boundary_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:calculator_runner",
+        "//mediapipe/framework/deps:file_path",
+        "//mediapipe/framework/formats:image_frame",
+        "//mediapipe/framework/formats:image_frame_opencv",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:opencv_core",
+        "//mediapipe/framework/port:opencv_imgcodecs",
+        "//mediapipe/framework/port:opencv_imgproc",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
+cc_library(
+    name = "face_to_region_calculator",
+    srcs = ["face_to_region_calculator.cc"],
+    deps = [
+        ":face_to_region_calculator_cc_proto",
+        "//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto",
+        "//mediapipe/examples/desktop/autoflip/quality:visual_scorer",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:detection_cc_proto",
+        "//mediapipe/framework/formats:image_frame",
+        "//mediapipe/framework/formats:image_frame_opencv",
+        "//mediapipe/framework/formats:location_data_cc_proto",
+        "//mediapipe/framework/port:opencv_core",
+        "//mediapipe/framework/port:opencv_imgproc",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/memory",
+    ],
+    alwayslink = 1,
+)
+
+proto_library(
+    name = "face_to_region_calculator_proto",
+    srcs = ["face_to_region_calculator.proto"],
+    deps = [
+        "//mediapipe/examples/desktop/autoflip/quality:visual_scorer_proto",
+        "//mediapipe/framework:calculator_proto",
+    ],
+)
+
+mediapipe_cc_proto_library(
+    name = "face_to_region_calculator_cc_proto",
+    srcs = ["face_to_region_calculator.proto"],
+    cc_deps = [
+        "//mediapipe/examples/desktop/autoflip/quality:visual_scorer_cc_proto",
+        "//mediapipe/framework:calculator_cc_proto",
+    ],
+    visibility = ["//mediapipe/examples:__subpackages__"],
+    deps = [":face_to_region_calculator_proto"],
+)
+
+cc_test(
+    name = "face_to_region_calculator_test",
+    srcs = ["face_to_region_calculator_test.cc"],
+    linkstatic = 1,
+    deps = [
+        ":face_to_region_calculator",
+        ":face_to_region_calculator_cc_proto",
+        "//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:calculator_runner",
+        "//mediapipe/framework/formats:detection_cc_proto",
+        "//mediapipe/framework/formats:image_frame",
+        "//mediapipe/framework/formats:image_frame_opencv",
+        "//mediapipe/framework/formats:location_data_cc_proto",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
+proto_library(
+    name = "localization_to_region_calculator_proto",
+    srcs = ["localization_to_region_calculator.proto"],
+    deps = [
+        "//mediapipe/framework:calculator_proto",
+    ],
+)
+
+mediapipe_cc_proto_library(
+    name = "localization_to_region_calculator_cc_proto",
+    srcs = ["localization_to_region_calculator.proto"],
+    cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
+    visibility = ["//mediapipe/examples:__subpackages__"],
+    deps = [":localization_to_region_calculator_proto"],
+)
+
+cc_library(
+    name = "localization_to_region_calculator",
+    srcs = ["localization_to_region_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":localization_to_region_calculator_cc_proto",
+        "//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:detection_cc_proto",
+        "//mediapipe/framework/formats:location_data_cc_proto",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/memory",
+    ],
+    alwayslink = 1,
+)
+
+cc_test(
+    name = "localization_to_region_calculator_test",
+    srcs = ["localization_to_region_calculator_test.cc"],
+    linkstatic = 1,
+    deps = [
+        ":localization_to_region_calculator",
+        ":localization_to_region_calculator_cc_proto",
+        "//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:calculator_runner",
+        "//mediapipe/framework/formats:detection_cc_proto",
+        "//mediapipe/framework/formats:location_data_cc_proto",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/strings",
+    ],
+)

mediapipe/examples/desktop/autoflip/calculators/border_detection_calculator.cc

@@ -0,0 +1,302 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// This Calculator takes an ImageFrame and scales it appropriately.
+
+#include <algorithm>
+#include <memory>
+#include <vector>
+
+#include "mediapipe/examples/desktop/autoflip/autoflip_messages.pb.h"
+#include "mediapipe/examples/desktop/autoflip/calculators/border_detection_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/image_frame.h"
+#include "mediapipe/framework/formats/image_frame_opencv.h"
+#include "mediapipe/framework/port/opencv_core_inc.h"
+#include "mediapipe/framework/port/opencv_imgproc_inc.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+
+using mediapipe::Adopt;
+using mediapipe::CalculatorBase;
+using mediapipe::ImageFrame;
+using mediapipe::PacketTypeSet;
+using mediapipe::autoflip::Border;
+
+constexpr char kDetectedBorders[] = "DETECTED_BORDERS";
+constexpr int kMinBorderDistance = 5;
+constexpr int kKMeansClusterCount = 4;
+constexpr int kMaxPixelsToProcess = 300000;
+constexpr char kVideoInputTag[] = "VIDEO";
+
+namespace mediapipe {
+namespace autoflip {
+
+namespace {
+
+// Sets rect values into a proto.
+void SetRect(const cv::Rect& region,
+             const Border::RelativePosition& relative_position, Border* part) {
+  part->mutable_border_position()->set_x(region.x);
+  part->mutable_border_position()->set_y(region.y);
+  part->mutable_border_position()->set_width(region.width);
+  part->mutable_border_position()->set_height(region.height);
+  part->set_relative_position(relative_position);
+}
+
+}  // namespace
+
+// This calculator takes a sequence of images (video) and detects solid color
+// borders as well as the dominant color of the non-border area.  This per-frame
+// information is passed to downstream calculators.
+class BorderDetectionCalculator : public CalculatorBase {
+ public:
+  BorderDetectionCalculator() : frame_width_(-1), frame_height_(-1) {}
+  ~BorderDetectionCalculator() override {}
+  BorderDetectionCalculator(const BorderDetectionCalculator&) = delete;
+  BorderDetectionCalculator& operator=(const BorderDetectionCalculator&) =
+      delete;
+
+  static mediapipe::Status GetContract(mediapipe::CalculatorContract* cc);
+  mediapipe::Status Open(mediapipe::CalculatorContext* cc) override;
+  mediapipe::Status Process(mediapipe::CalculatorContext* cc) override;
+
+ private:
+  // Given a color and image direction, check to see if a border of that color
+  // exists.
+  void DetectBorder(const cv::Mat& frame, const Color& color,
+                    const Border::RelativePosition& direction,
+                    StaticFeatures* features);
+
+  // Provide the percent this color shows up in a given image.
+  double ColorCount(const Color& mask_color, const cv::Mat& image) const;
+
+  // Set member vars (image size) and confirm no changes frame-to-frame.
+  mediapipe::Status SetAndCheckInputs(const cv::Mat& frame);
+
+  // Find the dominant color for a input image.
+  double FindDominantColor(const cv::Mat& image, Color* dominant_color);
+
+  // Frame width and height.
+  int frame_width_;
+  int frame_height_;
+
+  // Options for processing.
+  BorderDetectionCalculatorOptions options_;
+};
+REGISTER_CALCULATOR(BorderDetectionCalculator);
+
+::mediapipe::Status BorderDetectionCalculator::Open(
+    mediapipe::CalculatorContext* cc) {
+  options_ = cc->Options<BorderDetectionCalculatorOptions>();
+  RET_CHECK_LT(options_.vertical_search_distance(), 0.5)
+      << "Search distance must be less than half the full image.";
+  return ::mediapipe::OkStatus();
+}
+
+mediapipe::Status BorderDetectionCalculator::SetAndCheckInputs(
+    const cv::Mat& frame) {
+  if (frame_width_ < 0) {
+    frame_width_ = frame.cols;
+  }
+  if (frame_height_ < 0) {
+    frame_height_ = frame.rows;
+  }
+  RET_CHECK_EQ(frame.cols, frame_width_)
+      << "Input frame dimensions must remain constant throughout the video.";
+  RET_CHECK_EQ(frame.rows, frame_height_)
+      << "Input frame dimensions must remain constant throughout the video.";
+  RET_CHECK_EQ(frame.channels(), 3) << "Input video type must be 3-channel";
+  return ::mediapipe::OkStatus();
+}
+
+mediapipe::Status BorderDetectionCalculator::Process(
+    mediapipe::CalculatorContext* cc) {
+  if (!cc->Inputs().HasTag(kVideoInputTag) ||
+      cc->Inputs().Tag(kVideoInputTag).Value().IsEmpty()) {
+    return ::mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
+           << "Input tag VIDEO not set or empty at timestamp: "
+           << cc->InputTimestamp().Value();
+  }
+  cv::Mat frame = mediapipe::formats::MatView(
+      &cc->Inputs().Tag(kVideoInputTag).Get<ImageFrame>());
+  MP_RETURN_IF_ERROR(SetAndCheckInputs(frame));
+
+  // Initialize output and set default values.
+  std::unique_ptr<StaticFeatures> features =
+      absl::make_unique<StaticFeatures>();
+  features->mutable_non_static_area()->set_x(0);
+  features->mutable_non_static_area()->set_width(frame_width_);
+  features->mutable_non_static_area()->set_y(options_.default_padding_px());
+  features->mutable_non_static_area()->set_height(
+      std::max(0, frame_height_ - options_.default_padding_px() * 2));
+
+  // Check for border at the top of the frame.
+  Color seed_color_top;
+  FindDominantColor(frame(cv::Rect(0, 0, frame_width_, 1)), &seed_color_top);
+  DetectBorder(frame, seed_color_top, Border::TOP, features.get());
+
+  // Check for border at the bottom of the frame.
+  Color seed_color_bottom;
+  FindDominantColor(frame(cv::Rect(0, frame_height_ - 1, frame_width_, 1)),
+                    &seed_color_bottom);
+  DetectBorder(frame, seed_color_bottom, Border::BOTTOM, features.get());
+
+  // Check the non-border area for a dominant color.
+  cv::Mat non_static_frame = frame(
+      cv::Rect(features->non_static_area().x(), features->non_static_area().y(),
+               features->non_static_area().width(),
+               features->non_static_area().height()));
+  Color dominant_color_nonborder;
+  double dominant_color_percent =
+      FindDominantColor(non_static_frame, &dominant_color_nonborder);
+  if (dominant_color_percent > options_.solid_background_tol_perc()) {
+    auto* bg_color = features->mutable_solid_background();
+    bg_color->set_r(dominant_color_nonborder.r());
+    bg_color->set_g(dominant_color_nonborder.g());
+    bg_color->set_b(dominant_color_nonborder.b());
+  }
+
+  // Output result.
+  cc->Outputs()
+      .Tag(kDetectedBorders)
+      .AddPacket(Adopt(features.release()).At(cc->InputTimestamp()));
+
+  return ::mediapipe::OkStatus();
+}
+
+//  Find the dominant color within an image.
+double BorderDetectionCalculator::FindDominantColor(const cv::Mat& image_raw,
+                                                    Color* dominant_color) {
+  cv::Mat image;
+  if (image_raw.total() > kMaxPixelsToProcess) {
+    float resize = kMaxPixelsToProcess / static_cast<float>(image_raw.total());
+    cv::resize(image_raw, image, cv::Size(), resize, resize);
+  } else {
+    image = image_raw;
+  }
+
+  cv::Mat float_data, cluster, cluster_center;
+  image.convertTo(float_data, CV_32F);
+  cv::Mat reshaped = float_data.reshape(1, float_data.total());
+
+  cv::kmeans(reshaped, kKMeansClusterCount, cluster,
+             cv::TermCriteria(CV_TERMCRIT_ITER, 5, 1.0), 1,
+             cv::KMEANS_PP_CENTERS, cluster_center);
+
+  std::vector<int> count(kKMeansClusterCount, 0);
+  for (int i = 0; i < cluster.rows; i++) {
+    count[cluster.at<int>(i, 0)]++;
+  }
+  auto max_cluster_ptr = std::max_element(count.begin(), count.end());
+  double max_cluster_perc =
+      *max_cluster_ptr / static_cast<double>(cluster.rows);
+  int max_cluster_idx = std::distance(count.begin(), max_cluster_ptr);
+
+  dominant_color->set_r(cluster_center.at<float>(max_cluster_idx, 2));
+  dominant_color->set_g(cluster_center.at<float>(max_cluster_idx, 1));
+  dominant_color->set_b(cluster_center.at<float>(max_cluster_idx, 0));
+
+  return max_cluster_perc;
+}
+
+double BorderDetectionCalculator::ColorCount(const Color& mask_color,
+                                             const cv::Mat& image) const {
+  int background_count = 0;
+  for (int i = 0; i < image.rows; i++) {
+    const uint8* row_ptr = image.ptr<uint8>(i);
+    for (int j = 0; j < image.cols * 3; j += 3) {
+      if (std::abs(mask_color.r() - static_cast<int>(row_ptr[j + 2])) <=
+              options_.color_tolerance() &&
+          std::abs(mask_color.g() - static_cast<int>(row_ptr[j + 1])) <=
+              options_.color_tolerance() &&
+          std::abs(mask_color.b() - static_cast<int>(row_ptr[j])) <=
+              options_.color_tolerance()) {
+        background_count++;
+      }
+    }
+  }
+  return background_count / static_cast<double>(image.rows * image.cols);
+}
+
+void BorderDetectionCalculator::DetectBorder(
+    const cv::Mat& frame, const Color& color,
+    const Border::RelativePosition& direction, StaticFeatures* features) {
+  // Search the entire image until we find an object, or hit the max search
+  // distance.
+  int search_distance =
+      (direction == Border::TOP || direction == Border::BOTTOM) ? frame.rows
+                                                                : frame.cols;
+  search_distance *= options_.vertical_search_distance();
+
+  // Check if each next line has a dominant color that matches the given
+  // border color.
+  int last_border = -1;
+  for (int i = 0; i < search_distance; i++) {
+    cv::Rect current_row;
+    switch (direction) {
+      case Border::TOP:
+        current_row = cv::Rect(0, i, frame.cols, 1);
+        break;
+      case Border::BOTTOM:
+        current_row = cv::Rect(0, frame.rows - i - 1, frame.cols, 1);
+        break;
+    }
+    if (ColorCount(color, frame(current_row)) <
+        options_.border_color_pixel_perc()) {
+      break;
+    }
+    last_border = i;
+  }
+
+  // Reject results that are not borders (or too small).
+  if (last_border <= kMinBorderDistance || last_border == search_distance - 1) {
+    return;
+  }
+
+  // Apply defined padding.
+  last_border += options_.border_object_padding_px();
+
+  switch (direction) {
+    case Border::TOP:
+      SetRect(cv::Rect(0, 0, frame.cols, last_border), Border::TOP,
+              features->add_border());
+      features->mutable_non_static_area()->set_y(
+          last_border + features->non_static_area().y());
+      features->mutable_non_static_area()->set_height(
+          std::max(0, frame_height_ - (features->non_static_area().y() +
+                                       options_.default_padding_px())));
+      break;
+    case Border::BOTTOM:
+      SetRect(
+          cv::Rect(0, frame.rows - last_border - 1, frame.cols, last_border),
+          Border::BOTTOM, features->add_border());
+
+      features->mutable_non_static_area()->set_height(std::max(
+          0, frame.rows - (features->non_static_area().y() + last_border +
+                           options_.default_padding_px())));
+
+      break;
+  }
+}
+
+::mediapipe::Status BorderDetectionCalculator::GetContract(
+    mediapipe::CalculatorContract* cc) {
+  cc->Inputs().Tag(kVideoInputTag).Set<ImageFrame>();
+  cc->Outputs().Tag(kDetectedBorders).Set<StaticFeatures>();
+  return ::mediapipe::OkStatus();
+}
+
+}  // namespace autoflip
+}  // namespace mediapipe

mediapipe/examples/desktop/autoflip/calculators/border_detection_calculator.proto

@@ -0,0 +1,44 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe.autoflip;
+
+import "mediapipe/framework/calculator.proto";
+
+// Next tag: 7
+message BorderDetectionCalculatorOptions {
+  extend mediapipe.CalculatorOptions {
+    optional BorderDetectionCalculatorOptions ext = 276599815;
+  }
+  // Max difference in color to be considered the same (per rgb channel).
+  optional int32 color_tolerance = 1 [default = 6];
+
+  // Amount of padding to add around any object within the border that is
+  // resized to fit into the new border.
+  optional int32 border_object_padding_px = 2 [default = 5];
+
+  // Distance (as a percent of height) to search for a border.
+  optional float vertical_search_distance = 3 [default = .20];
+
+  // Percent of pixels matching border color to be a border
+  optional float border_color_pixel_perc = 4 [default = .995];
+
+  // Percent of pixels matching background to be a solid background frame
+  optional float solid_background_tol_perc = 5 [default = .5];
+
+  // Force a border of this size in pixels on top and bottom.
+  optional int32 default_padding_px = 6 [default = 0];
+}

mediapipe/examples/desktop/autoflip/calculators/border_detection_calculator_test.cc

@@ -0,0 +1,397 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "absl/strings/string_view.h"
+#include "mediapipe/examples/desktop/autoflip/autoflip_messages.pb.h"
+#include "mediapipe/examples/desktop/autoflip/calculators/border_detection_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/formats/image_frame.h"
+#include "mediapipe/framework/formats/image_frame_opencv.h"
+#include "mediapipe/framework/port/benchmark.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/opencv_core_inc.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/port/status_matchers.h"
+
+using mediapipe::Adopt;
+using mediapipe::CalculatorGraphConfig;
+using mediapipe::CalculatorRunner;
+using mediapipe::ImageFormat;
+using mediapipe::ImageFrame;
+using mediapipe::Packet;
+using mediapipe::PacketTypeSet;
+using mediapipe::ParseTextProtoOrDie;
+using mediapipe::Timestamp;
+using mediapipe::autoflip::Border;
+
+namespace mediapipe {
+namespace autoflip {
+namespace {
+
+const char kConfig[] = R"(
+    calculator: "BorderDetectionCalculator"
+    input_stream: "VIDEO:camera_frames"
+    output_stream: "DETECTED_BORDERS:regions"
+    options:{
+    [mediapipe.autoflip.BorderDetectionCalculatorOptions.ext]:{
+      border_object_padding_px: 0
+    }
+    })";
+
+const char kConfigPad[] = R"(
+    calculator: "BorderDetectionCalculator"
+    input_stream: "VIDEO:camera_frames"
+    output_stream: "DETECTED_BORDERS:regions"
+    options:{
+    [mediapipe.autoflip.BorderDetectionCalculatorOptions.ext]:{
+      default_padding_px: 10
+      border_object_padding_px: 0
+    }
+    })";
+
+const int kTestFrameWidth = 640;
+const int kTestFrameHeight = 480;
+
+const int kTestFrameLargeWidth = 1920;
+const int kTestFrameLargeHeight = 1080;
+
+const int kTestFrameWidthTall = 1200;
+const int kTestFrameHeightTall = 2001;
+
+TEST(BorderDetectionCalculatorTest, NoBorderTest) {
+  auto runner = ::absl::make_unique<CalculatorRunner>(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfig));
+
+  auto input_frame = ::absl::make_unique<ImageFrame>(
+      ImageFormat::SRGB, kTestFrameWidth, kTestFrameHeight);
+  cv::Mat input_mat = mediapipe::formats::MatView(input_frame.get());
+  input_mat.setTo(cv::Scalar(0, 0, 0));
+  runner->MutableInputs()->Tag("VIDEO").packets.push_back(
+      Adopt(input_frame.release()).At(Timestamp::PostStream()));
+
+  // Run the calculator.
+  MP_ASSERT_OK(runner->Run());
+
+  const std::vector<Packet>& output_packets =
+      runner->Outputs().Tag("DETECTED_BORDERS").packets;
+  ASSERT_EQ(1, output_packets.size());
+  const auto& static_features = output_packets[0].Get<StaticFeatures>();
+  ASSERT_EQ(0, static_features.border().size());
+  EXPECT_EQ(0, static_features.non_static_area().x());
+  EXPECT_EQ(0, static_features.non_static_area().y());
+  EXPECT_EQ(kTestFrameWidth, static_features.non_static_area().width());
+  EXPECT_EQ(kTestFrameHeight, static_features.non_static_area().height());
+  EXPECT_TRUE(static_features.has_solid_background());
+  EXPECT_EQ(0, static_features.solid_background().r());
+  EXPECT_EQ(0, static_features.solid_background().g());
+  EXPECT_EQ(0, static_features.solid_background().b());
+}
+
+TEST(BorderDetectionCalculatorTest, TopBorderTest) {
+  auto runner = ::absl::make_unique<CalculatorRunner>(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfig));
+
+  const int kTopBorderHeight = 50;
+
+  auto input_frame = ::absl::make_unique<ImageFrame>(
+      ImageFormat::SRGB, kTestFrameWidth, kTestFrameHeight);
+  cv::Mat input_mat = mediapipe::formats::MatView(input_frame.get());
+  input_mat.setTo(cv::Scalar(0, 0, 0));
+  cv::Mat sub_image =
+      input_mat(cv::Rect(0, 0, kTestFrameWidth, kTopBorderHeight));
+  sub_image.setTo(cv::Scalar(255, 0, 0));
+  runner->MutableInputs()->Tag("VIDEO").packets.push_back(
+      Adopt(input_frame.release()).At(Timestamp::PostStream()));
+
+  // Run the calculator.
+  MP_ASSERT_OK(runner->Run());
+
+  const std::vector<Packet>& output_packets =
+      runner->Outputs().Tag("DETECTED_BORDERS").packets;
+  ASSERT_EQ(1, output_packets.size());
+  const auto& static_features = output_packets[0].Get<StaticFeatures>();
+  ASSERT_EQ(1, static_features.border().size());
+  const auto& part = static_features.border(0);
+  EXPECT_EQ(part.border_position().x(), 0);
+  EXPECT_EQ(part.border_position().y(), 0);
+  EXPECT_EQ(part.border_position().width(), kTestFrameWidth);
+  EXPECT_LT(std::abs(part.border_position().height() - kTopBorderHeight), 2);
+  EXPECT_TRUE(static_features.has_solid_background());
+  EXPECT_EQ(0, static_features.solid_background().r());
+  EXPECT_EQ(0, static_features.solid_background().g());
+  EXPECT_EQ(0, static_features.solid_background().b());
+  EXPECT_EQ(0, static_features.non_static_area().x());
+  EXPECT_EQ(kTopBorderHeight - 1, static_features.non_static_area().y());
+  EXPECT_EQ(kTestFrameWidth, static_features.non_static_area().width());
+  EXPECT_EQ(kTestFrameHeight - kTopBorderHeight + 1,
+            static_features.non_static_area().height());
+}
+
+TEST(BorderDetectionCalculatorTest, TopBorderPadTest) {
+  auto runner = ::absl::make_unique<CalculatorRunner>(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfigPad));
+
+  const int kTopBorderHeight = 50;
+
+  auto input_frame = ::absl::make_unique<ImageFrame>(
+      ImageFormat::SRGB, kTestFrameWidth, kTestFrameHeight);
+  cv::Mat input_mat = mediapipe::formats::MatView(input_frame.get());
+  input_mat.setTo(cv::Scalar(0, 0, 0));
+  cv::Mat sub_image =
+      input_mat(cv::Rect(0, 0, kTestFrameWidth, kTopBorderHeight));
+  sub_image.setTo(cv::Scalar(255, 0, 0));
+  runner->MutableInputs()->Tag("VIDEO").packets.push_back(
+      Adopt(input_frame.release()).At(Timestamp::PostStream()));
+
+  // Run the calculator.
+  MP_ASSERT_OK(runner->Run());
+
+  const std::vector<Packet>& output_packets =
+      runner->Outputs().Tag("DETECTED_BORDERS").packets;
+  ASSERT_EQ(1, output_packets.size());
+  const auto& static_features = output_packets[0].Get<StaticFeatures>();
+  ASSERT_EQ(1, static_features.border().size());
+  const auto& part = static_features.border(0);
+  EXPECT_EQ(part.border_position().x(), 0);
+  EXPECT_EQ(part.border_position().y(), 0);
+  EXPECT_EQ(part.border_position().width(), kTestFrameWidth);
+  EXPECT_LT(std::abs(part.border_position().height() - kTopBorderHeight), 2);
+  EXPECT_TRUE(static_features.has_solid_background());
+  EXPECT_EQ(0, static_features.solid_background().r());
+  EXPECT_EQ(0, static_features.solid_background().g());
+  EXPECT_EQ(0, static_features.solid_background().b());
+  EXPECT_EQ(Border::TOP, part.relative_position());
+  EXPECT_EQ(0, static_features.non_static_area().x());
+  EXPECT_EQ(9 + kTopBorderHeight, static_features.non_static_area().y());
+  EXPECT_EQ(kTestFrameWidth, static_features.non_static_area().width());
+  EXPECT_EQ(kTestFrameHeight - 19 - kTopBorderHeight,
+            static_features.non_static_area().height());
+}
+
+TEST(BorderDetectionCalculatorTest, BottomBorderTest) {
+  auto runner = ::absl::make_unique<CalculatorRunner>(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfig));
+
+  const int kBottomBorderHeight = 50;
+
+  auto input_frame = ::absl::make_unique<ImageFrame>(
+      ImageFormat::SRGB, kTestFrameWidth, kTestFrameHeight);
+  cv::Mat input_mat = mediapipe::formats::MatView(input_frame.get());
+  input_mat.setTo(cv::Scalar(0, 0, 0));
+  cv::Mat bottom_image =
+      input_mat(cv::Rect(0, kTestFrameHeight - kBottomBorderHeight,
+                         kTestFrameWidth, kBottomBorderHeight));
+  bottom_image.setTo(cv::Scalar(255, 0, 0));
+  runner->MutableInputs()->Tag("VIDEO").packets.push_back(
+      Adopt(input_frame.release()).At(Timestamp::PostStream()));
+
+  // Run the calculator.
+  MP_ASSERT_OK(runner->Run());
+
+  const std::vector<Packet>& output_packets =
+      runner->Outputs().Tag("DETECTED_BORDERS").packets;
+  ASSERT_EQ(1, output_packets.size());
+  const auto& static_features = output_packets[0].Get<StaticFeatures>();
+  ASSERT_EQ(1, static_features.border().size());
+  const auto& part = static_features.border(0);
+  EXPECT_EQ(part.border_position().x(), 0);
+  EXPECT_EQ(part.border_position().y(), kTestFrameHeight - kBottomBorderHeight);
+  EXPECT_EQ(part.border_position().width(), kTestFrameWidth);
+  EXPECT_LT(std::abs(part.border_position().height() - kBottomBorderHeight), 2);
+  EXPECT_TRUE(static_features.has_solid_background());
+  EXPECT_EQ(0, static_features.solid_background().r());
+  EXPECT_EQ(0, static_features.solid_background().g());
+  EXPECT_EQ(0, static_features.solid_background().b());
+  EXPECT_EQ(Border::BOTTOM, part.relative_position());
+}
+
+TEST(BorderDetectionCalculatorTest, TopBottomBorderTest) {
+  auto runner = ::absl::make_unique<CalculatorRunner>(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfig));
+
+  const int kBottomBorderHeight = 50;
+  const int kTopBorderHeight = 25;
+
+  auto input_frame = ::absl::make_unique<ImageFrame>(
+      ImageFormat::SRGB, kTestFrameWidth, kTestFrameHeight);
+  cv::Mat input_mat = mediapipe::formats::MatView(input_frame.get());
+  input_mat.setTo(cv::Scalar(0, 0, 0));
+  cv::Mat top_image =
+      input_mat(cv::Rect(0, 0, kTestFrameWidth, kTopBorderHeight));
+  top_image.setTo(cv::Scalar(0, 255, 0));
+  cv::Mat bottom_image =
+      input_mat(cv::Rect(0, kTestFrameHeight - kBottomBorderHeight,
+                         kTestFrameWidth, kBottomBorderHeight));
+  bottom_image.setTo(cv::Scalar(255, 0, 0));
+  runner->MutableInputs()->Tag("VIDEO").packets.push_back(
+      Adopt(input_frame.release()).At(Timestamp::PostStream()));
+
+  // Run the calculator.
+  MP_ASSERT_OK(runner->Run());
+
+  const std::vector<Packet>& output_packets =
+      runner->Outputs().Tag("DETECTED_BORDERS").packets;
+  ASSERT_EQ(1, output_packets.size());
+  const auto& static_features = output_packets[0].Get<StaticFeatures>();
+  ASSERT_EQ(2, static_features.border().size());
+  auto part = static_features.border(0);
+  EXPECT_EQ(part.border_position().x(), 0);
+  EXPECT_EQ(part.border_position().y(), 0);
+  EXPECT_EQ(part.border_position().width(), kTestFrameWidth);
+  EXPECT_LT(std::abs(part.border_position().height() - kTopBorderHeight), 2);
+  EXPECT_TRUE(static_features.has_solid_background());
+  EXPECT_EQ(0, static_features.solid_background().r());
+  EXPECT_EQ(0, static_features.solid_background().g());
+  EXPECT_EQ(0, static_features.solid_background().b());
+  EXPECT_EQ(0, static_features.non_static_area().x());
+  EXPECT_EQ(kTopBorderHeight - 1, static_features.non_static_area().y());
+  EXPECT_EQ(kTestFrameWidth, static_features.non_static_area().width());
+  EXPECT_EQ(kTestFrameHeight - kTopBorderHeight - kBottomBorderHeight + 2,
+            static_features.non_static_area().height());
+  EXPECT_EQ(Border::TOP, part.relative_position());
+
+  part = static_features.border(1);
+  EXPECT_EQ(part.border_position().x(), 0);
+  EXPECT_EQ(part.border_position().y(), kTestFrameHeight - kBottomBorderHeight);
+  EXPECT_EQ(part.border_position().width(), kTestFrameWidth);
+  EXPECT_LT(std::abs(part.border_position().height() - kBottomBorderHeight), 2);
+  EXPECT_EQ(Border::BOTTOM, part.relative_position());
+}
+
+TEST(BorderDetectionCalculatorTest, TopBottomBorderTestAspect2) {
+  auto runner = ::absl::make_unique<CalculatorRunner>(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfig));
+
+  const int kBottomBorderHeight = 50;
+  const int kTopBorderHeight = 25;
+
+  auto input_frame = ::absl::make_unique<ImageFrame>(
+      ImageFormat::SRGB, kTestFrameWidthTall, kTestFrameHeightTall);
+  cv::Mat input_mat = mediapipe::formats::MatView(input_frame.get());
+  input_mat.setTo(cv::Scalar(0, 0, 0));
+  cv::Mat top_image =
+      input_mat(cv::Rect(0, 0, kTestFrameWidthTall, kTopBorderHeight));
+  top_image.setTo(cv::Scalar(0, 255, 0));
+  cv::Mat bottom_image =
+      input_mat(cv::Rect(0, kTestFrameHeightTall - kBottomBorderHeight,
+                         kTestFrameWidthTall, kBottomBorderHeight));
+  bottom_image.setTo(cv::Scalar(255, 0, 0));
+  runner->MutableInputs()->Tag("VIDEO").packets.push_back(
+      Adopt(input_frame.release()).At(Timestamp::PostStream()));
+
+  // Run the calculator.
+  MP_ASSERT_OK(runner->Run());
+
+  const std::vector<Packet>& output_packets =
+      runner->Outputs().Tag("DETECTED_BORDERS").packets;
+  ASSERT_EQ(1, output_packets.size());
+  const auto& static_features = output_packets[0].Get<StaticFeatures>();
+  ASSERT_EQ(2, static_features.border().size());
+  auto part = static_features.border(0);
+  EXPECT_EQ(part.border_position().x(), 0);
+  EXPECT_EQ(part.border_position().y(), 0);
+  EXPECT_EQ(part.border_position().width(), kTestFrameWidthTall);
+  EXPECT_LT(std::abs(part.border_position().height() - kTopBorderHeight), 2);
+  EXPECT_TRUE(static_features.has_solid_background());
+  EXPECT_EQ(0, static_features.solid_background().r());
+  EXPECT_EQ(0, static_features.solid_background().g());
+  EXPECT_EQ(0, static_features.solid_background().b());
+  EXPECT_EQ(Border::TOP, part.relative_position());
+
+  part = static_features.border(1);
+  EXPECT_EQ(part.border_position().x(), 0);
+  EXPECT_EQ(part.border_position().y(),
+            kTestFrameHeightTall - kBottomBorderHeight);
+  EXPECT_EQ(part.border_position().width(), kTestFrameWidthTall);
+  EXPECT_LT(std::abs(part.border_position().height() - kBottomBorderHeight), 2);
+  EXPECT_TRUE(static_features.has_solid_background());
+  EXPECT_EQ(0, static_features.solid_background().r());
+  EXPECT_EQ(0, static_features.solid_background().g());
+  EXPECT_EQ(0, static_features.solid_background().b());
+  EXPECT_EQ(Border::BOTTOM, part.relative_position());
+}
+
+TEST(BorderDetectionCalculatorTest, DominantColor) {
+  CalculatorGraphConfig::Node node =
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfigPad);
+  node.mutable_options()
+      ->MutableExtension(BorderDetectionCalculatorOptions::ext)
+      ->set_solid_background_tol_perc(.25);
+
+  auto runner = ::absl::make_unique<CalculatorRunner>(node);
+
+  auto input_frame = ::absl::make_unique<ImageFrame>(
+      ImageFormat::SRGB, kTestFrameWidth, kTestFrameHeight);
+  cv::Mat input_mat = mediapipe::formats::MatView(input_frame.get());
+  input_mat.setTo(cv::Scalar(0, 0, 0));
+
+  cv::Mat sub_image = input_mat(cv::Rect(
+      kTestFrameWidth / 2, 0, kTestFrameWidth / 2, kTestFrameHeight / 2));
+  sub_image.setTo(cv::Scalar(0, 255, 0));
+
+  sub_image = input_mat(cv::Rect(0, kTestFrameHeight / 2, kTestFrameWidth / 2,
+                                 kTestFrameHeight / 2));
+  sub_image.setTo(cv::Scalar(0, 0, 255));
+
+  sub_image =
+      input_mat(cv::Rect(0, 0, kTestFrameWidth / 2 + 50, kTestFrameHeight / 2));
+  sub_image.setTo(cv::Scalar(255, 0, 0));
+
+  runner->MutableInputs()->Tag("VIDEO").packets.push_back(
+      Adopt(input_frame.release()).At(Timestamp::PostStream()));
+
+  // Run the calculator.
+  MP_ASSERT_OK(runner->Run());
+
+  const std::vector<Packet>& output_packets =
+      runner->Outputs().Tag("DETECTED_BORDERS").packets;
+  ASSERT_EQ(1, output_packets.size());
+  const auto& static_features = output_packets[0].Get<StaticFeatures>();
+  ASSERT_EQ(0, static_features.border().size());
+  ASSERT_TRUE(static_features.has_solid_background());
+  EXPECT_EQ(0, static_features.solid_background().r());
+  EXPECT_EQ(0, static_features.solid_background().g());
+  EXPECT_EQ(255, static_features.solid_background().b());
+}
+
+void BM_Large(benchmark::State& state) {
+  for (auto _ : state) {
+    auto runner = ::absl::make_unique<CalculatorRunner>(
+        ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfig));
+
+    const int kTopBorderHeight = 50;
+
+    auto input_frame = ::absl::make_unique<ImageFrame>(
+        ImageFormat::SRGB, kTestFrameLargeWidth, kTestFrameLargeHeight);
+    cv::Mat input_mat = mediapipe::formats::MatView(input_frame.get());
+    input_mat.setTo(cv::Scalar(0, 0, 0));
+    cv::Mat sub_image =
+        input_mat(cv::Rect(0, 0, kTestFrameLargeWidth, kTopBorderHeight));
+    sub_image.setTo(cv::Scalar(255, 0, 0));
+    runner->MutableInputs()->Tag("VIDEO").packets.push_back(
+        Adopt(input_frame.release()).At(Timestamp::PostStream()));
+
+    // Run the calculator.
+    MP_ASSERT_OK(runner->Run());
+  }
+}
+BENCHMARK(BM_Large);
+
+}  // namespace
+}  // namespace autoflip
+}  // namespace mediapipe

mediapipe/examples/desktop/autoflip/calculators/face_to_region_calculator.cc

@@ -0,0 +1,269 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <algorithm>
+#include <memory>
+
+#include "mediapipe/examples/desktop/autoflip/autoflip_messages.pb.h"
+#include "mediapipe/examples/desktop/autoflip/calculators/face_to_region_calculator.pb.h"
+#include "mediapipe/examples/desktop/autoflip/quality/visual_scorer.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/detection.pb.h"
+#include "mediapipe/framework/formats/image_frame.h"
+#include "mediapipe/framework/formats/image_frame_opencv.h"
+#include "mediapipe/framework/formats/location_data.pb.h"
+#include "mediapipe/framework/port/opencv_core_inc.h"
+#include "mediapipe/framework/port/opencv_imgproc_inc.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/port/status_builder.h"
+
+namespace mediapipe {
+namespace autoflip {
+
+// This calculator converts detected faces to SalientRegion protos that can be
+// used for downstream processing. Each SalientRegion is scored using image
+// cues. Scoring can be controlled through
+// FaceToRegionCalculator::scorer_options.
+// Example:
+//    calculator: "FaceToRegionCalculator"
+//    input_stream: "VIDEO:frames"
+//    input_stream: "FACES:faces"
+//    output_stream: "REGIONS:regions"
+//    options:{
+//      [mediapipe.autoflip.FaceToRegionCalculatorOptions.ext]:{
+//        export_individual_face_landmarks: false
+//        export_whole_face: true
+//      }
+//    }
+//
+class FaceToRegionCalculator : public CalculatorBase {
+ public:
+  FaceToRegionCalculator();
+  ~FaceToRegionCalculator() override {}
+  FaceToRegionCalculator(const FaceToRegionCalculator&) = delete;
+  FaceToRegionCalculator& operator=(const FaceToRegionCalculator&) = delete;
+
+  static ::mediapipe::Status GetContract(mediapipe::CalculatorContract* cc);
+  ::mediapipe::Status Open(mediapipe::CalculatorContext* cc) override;
+  ::mediapipe::Status Process(mediapipe::CalculatorContext* cc) override;
+
+ private:
+  double NormalizeX(const int pixel);
+  double NormalizeY(const int pixel);
+  // Extend the given SalientRegion to include the given point.
+  void ExtendSalientRegionWithPoint(const float x, const float y,
+                                    SalientRegion* region);
+  // Calculator options.
+  FaceToRegionCalculatorOptions options_;
+
+  // A scorer used to assign weights to faces.
+  std::unique_ptr<VisualScorer> scorer_;
+  // Dimensions of video frame
+  int frame_width_;
+  int frame_height_;
+};
+REGISTER_CALCULATOR(FaceToRegionCalculator);
+
+FaceToRegionCalculator::FaceToRegionCalculator() {}
+
+::mediapipe::Status FaceToRegionCalculator::GetContract(
+    mediapipe::CalculatorContract* cc) {
+  cc->Inputs().Tag("VIDEO").Set<ImageFrame>();
+  cc->Inputs().Tag("FACES").Set<std::vector<mediapipe::Detection>>();
+  cc->Outputs().Tag("REGIONS").Set<DetectionSet>();
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status FaceToRegionCalculator::Open(
+    mediapipe::CalculatorContext* cc) {
+  options_ = cc->Options<FaceToRegionCalculatorOptions>();
+  scorer_ = absl::make_unique<VisualScorer>(options_.scorer_options());
+  frame_width_ = -1;
+  frame_height_ = -1;
+  return ::mediapipe::OkStatus();
+}
+
+inline double FaceToRegionCalculator::NormalizeX(const int pixel) {
+  return pixel / static_cast<double>(frame_width_);
+}
+
+inline double FaceToRegionCalculator::NormalizeY(const int pixel) {
+  return pixel / static_cast<double>(frame_height_);
+}
+
+void FaceToRegionCalculator::ExtendSalientRegionWithPoint(
+    const float x, const float y, SalientRegion* region) {
+  auto* location = region->mutable_location_normalized();
+  if (!location->has_width()) {
+    location->set_width(NormalizeX(1));
+  } else if (x < location->x()) {
+    location->set_width(location->width() + location->x() - x);
+  } else if (x > location->x() + location->width()) {
+    location->set_width(x - location->x());
+  }
+  if (!location->has_height()) {
+    location->set_height(NormalizeY(1));
+  } else if (y < location->y()) {
+    location->set_height(location->height() + location->y() - y);
+  } else if (y > location->y() + location->height()) {
+    location->set_height(y - location->y());
+  }
+
+  if (!location->has_x()) {
+    location->set_x(x);
+  } else {
+    location->set_x(std::min(location->x(), x));
+  }
+  if (!location->has_y()) {
+    location->set_y(y);
+  } else {
+    location->set_y(std::min(location->y(), y));
+  }
+}
+
+::mediapipe::Status FaceToRegionCalculator::Process(
+    mediapipe::CalculatorContext* cc) {
+  if (cc->Inputs().Tag("VIDEO").Value().IsEmpty()) {
+    return ::mediapipe::UnknownErrorBuilder(MEDIAPIPE_LOC) << "No VIDEO input.";
+  }
+
+  cv::Mat frame =
+      mediapipe::formats::MatView(&cc->Inputs().Tag("VIDEO").Get<ImageFrame>());
+  frame_width_ = frame.cols;
+  frame_height_ = frame.rows;
+
+  auto region_set = ::absl::make_unique<DetectionSet>();
+  if (!cc->Inputs().Tag("FACES").Value().IsEmpty()) {
+    const auto& input_faces =
+        cc->Inputs().Tag("FACES").Get<std::vector<mediapipe::Detection>>();
+
+    for (const auto& input_face : input_faces) {
+      RET_CHECK(input_face.location_data().format() ==
+                mediapipe::LocationData::RELATIVE_BOUNDING_BOX)
+          << "Face detection input is lacking required relative_bounding_box()";
+      // 6 landmarks should be provided, ordered as:
+      // Left eye, Right eye, Nose tip, Mouth center, Left ear tragion, Right
+      // ear tragion.
+      RET_CHECK(input_face.location_data().relative_keypoints().size() == 6)
+          << "Face detection input expected 6 keypoints, has "
+          << input_face.location_data().relative_keypoints().size();
+
+      const auto& location = input_face.location_data().relative_bounding_box();
+
+      // Reduce region size to only contain parts of the image in frame.
+      float x = std::max(0.0f, location.xmin());
+      float y = std::max(0.0f, location.ymin());
+      float width =
+          std::min(location.width() - abs(x - location.xmin()), 1 - x);
+      float height =
+          std::min(location.height() - abs(y - location.ymin()), 1 - y);
+
+      // Convert the face to a region.
+      if (options_.export_whole_face()) {
+        SalientRegion* region = region_set->add_detections();
+        region->mutable_location_normalized()->set_x(x);
+        region->mutable_location_normalized()->set_y(y);
+        region->mutable_location_normalized()->set_width(width);
+        region->mutable_location_normalized()->set_height(height);
+        region->mutable_signal_type()->set_standard(SignalType::FACE_FULL);
+
+        // Score the face based on image cues.
+        float visual_score = 1.0f;
+        if (options_.use_visual_scorer()) {
+          MP_RETURN_IF_ERROR(
+              scorer_->CalculateScore(frame, *region, &visual_score));
+        }
+        region->set_score(visual_score);
+      }
+
+      // Generate two more output regions from important face landmarks. One
+      // includes all exterior landmarks, such as ears and chin, and the
+      // other includes only interior landmarks, such as the eye edges and the
+      // mouth.
+      SalientRegion core_landmark_region, all_landmark_region;
+      // Keypoints are ordered: Left Eye, Right Eye, Nose Tip, Mouth Center,
+      // Left Ear Tragion, Right Ear Tragion.
+
+      // Set 'core' landmarks (Left Eye, Right Eye, Nose Tip, Mouth Center)
+      for (int i = 0; i < 4; i++) {
+        const auto& keypoint = input_face.location_data().relative_keypoints(i);
+        if (options_.export_individual_face_landmarks()) {
+          SalientRegion* region = region_set->add_detections();
+          region->mutable_location_normalized()->set_x(keypoint.x());
+          region->mutable_location_normalized()->set_y(keypoint.y());
+          region->mutable_location_normalized()->set_width(NormalizeX(1));
+          region->mutable_location_normalized()->set_height(NormalizeY(1));
+          region->mutable_signal_type()->set_standard(
+              SignalType::FACE_LANDMARK);
+        }
+
+        // Extend the core/full landmark regions to include the new
+        ExtendSalientRegionWithPoint(keypoint.x(), keypoint.y(),
+                                     &core_landmark_region);
+        ExtendSalientRegionWithPoint(keypoint.x(), keypoint.y(),
+                                     &all_landmark_region);
+      }
+      // Set 'all' landmarks (Left Ear Tragion, Right Ear Tragion + core)
+      for (int i = 4; i < 6; i++) {
+        const auto& keypoint = input_face.location_data().relative_keypoints(i);
+        if (options_.export_individual_face_landmarks()) {
+          SalientRegion* region = region_set->add_detections();
+          region->mutable_location()->set_x(keypoint.x());
+          region->mutable_location()->set_y(keypoint.y());
+          region->mutable_location()->set_width(NormalizeX(1));
+          region->mutable_location()->set_height(NormalizeY(1));
+          region->mutable_signal_type()->set_standard(
+              SignalType::FACE_LANDMARK);
+        }
+
+        // Extend the full landmark region to include the new landmark.
+        ExtendSalientRegionWithPoint(keypoint.x(), keypoint.y(),
+                                     &all_landmark_region);
+      }
+
+      // Generate scores for the landmark bboxes and export them.
+      if (options_.export_bbox_from_landmarks() &&
+          core_landmark_region.has_location_normalized()) {  // Not empty.
+        float visual_score = 1.0f;
+        if (options_.use_visual_scorer()) {
+          MP_RETURN_IF_ERROR(scorer_->CalculateScore(
+              frame, core_landmark_region, &visual_score));
+        }
+        core_landmark_region.set_score(visual_score);
+        core_landmark_region.mutable_signal_type()->set_standard(
+            SignalType::FACE_CORE_LANDMARKS);
+        *region_set->add_detections() = core_landmark_region;
+      }
+      if (options_.export_bbox_from_landmarks() &&
+          all_landmark_region.has_location_normalized()) {  // Not empty.
+        float visual_score = 1.0f;
+        if (options_.use_visual_scorer()) {
+          MP_RETURN_IF_ERROR(scorer_->CalculateScore(frame, all_landmark_region,
+                                                     &visual_score));
+        }
+        all_landmark_region.set_score(visual_score);
+        all_landmark_region.mutable_signal_type()->set_standard(
+            SignalType::FACE_ALL_LANDMARKS);
+        *region_set->add_detections() = all_landmark_region;
+      }
+    }
+  }
+  cc->Outputs().Tag("REGIONS").Add(region_set.release(), cc->InputTimestamp());
+
+  return ::mediapipe::OkStatus();
+}
+
+}  // namespace autoflip
+}  // namespace mediapipe

mediapipe/examples/desktop/autoflip/calculators/face_to_region_calculator.proto

@@ -0,0 +1,50 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe.autoflip;
+
+import "mediapipe/examples/desktop/autoflip/quality/visual_scorer.proto";
+import "mediapipe/framework/calculator.proto";
+
+// Next tag: 6
+message FaceToRegionCalculatorOptions {
+  extend mediapipe.CalculatorOptions {
+    optional FaceToRegionCalculatorOptions ext = 282401234;
+  }
+
+  // Options for generating a score for the entire face from its visual
+  // appearance. The generated score is used to modulate the detection scores
+  // for whole face and/or landmark bbox region types.
+  optional VisualScorerOptions scorer_options = 1;
+
+  // If true, export the large face bounding box generated by the face tracker.
+  // This bounding box is generally larger than the actual face and relatively
+  // inaccurate.
+  optional bool export_whole_face = 2 [default = false];
+
+  // If true, export a number of individual face landmarks (eyes, nose, mouth,
+  // ears etc) as separate SalientRegion protos.
+  optional bool export_individual_face_landmarks = 3 [default = false];
+
+  // If true, export two bounding boxes from landmarks (one for the core face
+  // landmarks like eyes and nose, and one for extended landmarks including ears
+  // and chin).
+  optional bool export_bbox_from_landmarks = 4 [default = true];
+
+  // If true, generate a score from the appearance of the face and use it to
+  // modulate the detection scores for whole face and/or landmark bboxes.
+  optional bool use_visual_scorer = 5 [default = true];
+}

mediapipe/examples/desktop/autoflip/calculators/face_to_region_calculator_test.cc

@@ -0,0 +1,247 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "absl/strings/string_view.h"
+#include "mediapipe/examples/desktop/autoflip/autoflip_messages.pb.h"
+#include "mediapipe/examples/desktop/autoflip/calculators/face_to_region_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/formats/detection.pb.h"
+#include "mediapipe/framework/formats/image_frame.h"
+#include "mediapipe/framework/formats/image_frame_opencv.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/port/status_matchers.h"
+
+using mediapipe::Detection;
+
+namespace mediapipe {
+namespace autoflip {
+namespace {
+
+const char kConfig[] = R"(
+    calculator: "FaceToRegionCalculator"
+    input_stream: "VIDEO:frames"
+    input_stream: "FACES:faces"
+    output_stream: "REGIONS:regions"
+    )";
+
+const char kFace1[] = R"(location_data {
+           format: RELATIVE_BOUNDING_BOX
+           relative_bounding_box {
+             xmin: -0.00375
+             ymin: 0.003333
+             width: 0.125
+             height: 0.33333
+           }
+           relative_keypoints { x: 0.03125 y: 0.05 }
+           relative_keypoints { x: 0.0875 y: 0.0666666 }
+           relative_keypoints { x: 0.03125 y: 0.05 }
+           relative_keypoints { x: 0.0875 y: 0.0666666 }
+           relative_keypoints { x: 0.0250 y: 0.0666666 }
+           relative_keypoints { x: 0.0950 y: 0.0666666 }
+         })";
+
+const char kFace2[] = R"(location_data {
+           format: RELATIVE_BOUNDING_BOX
+           relative_bounding_box {
+             xmin: 0.0025
+             ymin: 0.005
+             width: 0.25
+             height: 0.5
+           }
+           relative_keypoints { x: 0 y: 0 }
+           relative_keypoints { x: 0 y: 0 }
+           relative_keypoints { x: 0 y: 0 }
+           relative_keypoints { x: 0 y: 0 }
+           relative_keypoints { x: 0 y: 0 }
+           relative_keypoints { x: 0 y: 0 }
+         })";
+
+const char kFace3[] = R"(location_data {
+           format: RELATIVE_BOUNDING_BOX
+           relative_bounding_box {
+             xmin: 0.0
+             ymin: 0.0
+             width: 0.5
+             height: 0.5
+           }
+           relative_keypoints { x: 0 y: 0 }
+           relative_keypoints { x: 0 y: 0 }
+           relative_keypoints { x: 0 y: 0 }
+           relative_keypoints { x: 0 y: 0 }
+           relative_keypoints { x: 0 y: 0 }
+           relative_keypoints { x: 0 y: 0 }
+         })";
+
+void SetInputs(CalculatorRunner* runner,
+               const std::vector<std::string>& faces) {
+  // Setup an input video frame.
+  auto input_frame =
+      ::absl::make_unique<ImageFrame>(ImageFormat::SRGB, 800, 600);
+  runner->MutableInputs()->Tag("VIDEO").packets.push_back(
+      Adopt(input_frame.release()).At(Timestamp::PostStream()));
+  // Setup two faces as input.
+  auto input_faces = ::absl::make_unique<std::vector<Detection>>();
+  // A face with landmarks.
+  for (const auto& face : faces) {
+    input_faces->push_back(ParseTextProtoOrDie<Detection>(face));
+  }
+  runner->MutableInputs()->Tag("FACES").packets.push_back(
+      Adopt(input_faces.release()).At(Timestamp::PostStream()));
+}
+
+CalculatorGraphConfig::Node MakeConfig(bool whole_face, bool landmarks,
+                                       bool bb_from_landmarks) {
+  auto config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfig);
+
+  config.mutable_options()
+      ->MutableExtension(FaceToRegionCalculatorOptions::ext)
+      ->set_export_whole_face(whole_face);
+
+  config.mutable_options()
+      ->MutableExtension(FaceToRegionCalculatorOptions::ext)
+      ->set_export_individual_face_landmarks(landmarks);
+
+  config.mutable_options()
+      ->MutableExtension(FaceToRegionCalculatorOptions::ext)
+      ->set_export_bbox_from_landmarks(bb_from_landmarks);
+
+  return config;
+}
+
+TEST(FaceToRegionCalculatorTest, FaceFullTypeSize) {
+  // Setup test
+  auto runner =
+      ::absl::make_unique<CalculatorRunner>(MakeConfig(true, false, false));
+  SetInputs(runner.get(), {kFace1, kFace2});
+
+  // Run the calculator.
+  MP_ASSERT_OK(runner->Run());
+
+  // Check the output regions.
+  const std::vector<Packet>& output_packets =
+      runner->Outputs().Tag("REGIONS").packets;
+  ASSERT_EQ(1, output_packets.size());
+
+  const auto& regions = output_packets[0].Get<DetectionSet>();
+  ASSERT_EQ(2, regions.detections().size());
+  auto face_1 = regions.detections(0);
+  EXPECT_EQ(face_1.signal_type().standard(), SignalType::FACE_FULL);
+  EXPECT_FLOAT_EQ(face_1.location_normalized().x(), 0);
+  EXPECT_FLOAT_EQ(face_1.location_normalized().y(), 0.003333);
+  EXPECT_FLOAT_EQ(face_1.location_normalized().width(), 0.12125);
+  EXPECT_FLOAT_EQ(face_1.location_normalized().height(), 0.33333);
+  EXPECT_FLOAT_EQ(face_1.score(), 0.040214583);
+
+  auto face_2 = regions.detections(1);
+  EXPECT_EQ(face_2.signal_type().standard(), SignalType::FACE_FULL);
+  EXPECT_FLOAT_EQ(face_2.location_normalized().x(), 0.0025);
+  EXPECT_FLOAT_EQ(face_2.location_normalized().y(), 0.005);
+  EXPECT_FLOAT_EQ(face_2.location_normalized().width(), 0.25);
+  EXPECT_FLOAT_EQ(face_2.location_normalized().height(), 0.5);
+  EXPECT_FLOAT_EQ(face_2.score(), 0.125);
+}
+
+TEST(FaceToRegionCalculatorTest, FaceLandmarksTypeSize) {
+  // Setup test
+  auto runner =
+      ::absl::make_unique<CalculatorRunner>(MakeConfig(false, true, false));
+  SetInputs(runner.get(), {kFace1});
+
+  // Run the calculator.
+  MP_ASSERT_OK(runner->Run());
+
+  // Check the output regions.
+  const std::vector<Packet>& output_packets =
+      runner->Outputs().Tag("REGIONS").packets;
+  ASSERT_EQ(1, output_packets.size());
+
+  const auto& regions = output_packets[0].Get<DetectionSet>();
+  ASSERT_EQ(6, regions.detections().size());
+  auto landmark_1 = regions.detections(0);
+  EXPECT_EQ(landmark_1.signal_type().standard(), SignalType::FACE_LANDMARK);
+  EXPECT_FLOAT_EQ(landmark_1.location_normalized().x(), 0.03125);
+  EXPECT_FLOAT_EQ(landmark_1.location_normalized().y(), 0.05);
+  EXPECT_FLOAT_EQ(landmark_1.location_normalized().width(), 0.00125);
+  EXPECT_FLOAT_EQ(landmark_1.location_normalized().height(), 0.0016666667);
+
+  auto landmark_2 = regions.detections(1);
+  EXPECT_EQ(landmark_2.signal_type().standard(), SignalType::FACE_LANDMARK);
+  EXPECT_FLOAT_EQ(landmark_2.location_normalized().x(), 0.0875);
+  EXPECT_FLOAT_EQ(landmark_2.location_normalized().y(), 0.0666666);
+  EXPECT_FLOAT_EQ(landmark_2.location_normalized().width(), 0.00125);
+  EXPECT_FLOAT_EQ(landmark_2.location_normalized().height(), 0.0016666667);
+}
+
+TEST(FaceToRegionCalculatorTest, FaceLandmarksBox) {
+  // Setup test
+  auto runner =
+      ::absl::make_unique<CalculatorRunner>(MakeConfig(false, false, true));
+  SetInputs(runner.get(), {kFace1});
+
+  // Run the calculator.
+  MP_ASSERT_OK(runner->Run());
+
+  // Check the output regions.
+  const std::vector<Packet>& output_packets =
+      runner->Outputs().Tag("REGIONS").packets;
+  ASSERT_EQ(1, output_packets.size());
+
+  const auto& regions = output_packets[0].Get<DetectionSet>();
+  ASSERT_EQ(2, regions.detections().size());
+  auto landmark_1 = regions.detections(0);
+  EXPECT_EQ(landmark_1.signal_type().standard(),
+            SignalType::FACE_CORE_LANDMARKS);
+  EXPECT_FLOAT_EQ(landmark_1.location_normalized().x(), 0.03125);
+  EXPECT_FLOAT_EQ(landmark_1.location_normalized().y(), 0.05);
+  EXPECT_FLOAT_EQ(landmark_1.location_normalized().width(), 0.056249999);
+  EXPECT_FLOAT_EQ(landmark_1.location_normalized().height(), 0.016666602);
+  EXPECT_FLOAT_EQ(landmark_1.score(), 0.00084375002);
+
+  auto landmark_2 = regions.detections(1);
+  EXPECT_EQ(landmark_2.signal_type().standard(),
+            SignalType::FACE_ALL_LANDMARKS);
+  EXPECT_FLOAT_EQ(landmark_2.location_normalized().x(), 0.025);
+  EXPECT_FLOAT_EQ(landmark_2.location_normalized().y(), 0.050000001);
+  EXPECT_FLOAT_EQ(landmark_2.location_normalized().width(), 0.07);
+  EXPECT_FLOAT_EQ(landmark_2.location_normalized().height(), 0.016666602);
+  EXPECT_FLOAT_EQ(landmark_2.score(), 0.00105);
+}
+
+TEST(FaceToRegionCalculatorTest, FaceScore) {
+  // Setup test
+  auto runner =
+      ::absl::make_unique<CalculatorRunner>(MakeConfig(true, false, false));
+  SetInputs(runner.get(), {kFace3});
+
+  // Run the calculator.
+  MP_ASSERT_OK(runner->Run());
+
+  // Check the output regions.
+  const std::vector<Packet>& output_packets =
+      runner->Outputs().Tag("REGIONS").packets;
+  ASSERT_EQ(1, output_packets.size());
+  const auto& regions = output_packets[0].Get<DetectionSet>();
+  ASSERT_EQ(1, regions.detections().size());
+  auto landmark_1 = regions.detections(0);
+  EXPECT_FLOAT_EQ(landmark_1.score(), 0.25);
+}
+
+}  // namespace
+}  // namespace autoflip
+}  // namespace mediapipe

mediapipe/examples/desktop/autoflip/calculators/localization_to_region_calculator.cc

@@ -0,0 +1,126 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <memory>
+#include <string>
+
+#include "absl/memory/memory.h"
+#include "mediapipe/examples/desktop/autoflip/autoflip_messages.pb.h"
+#include "mediapipe/examples/desktop/autoflip/calculators/localization_to_region_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/detection.pb.h"
+#include "mediapipe/framework/formats/location_data.pb.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+namespace autoflip {
+
+// This calculator converts detections from ObjectLocalizationCalculator to
+// SalientRegion protos that can be used for downstream processing.
+class LocalizationToRegionCalculator : public mediapipe::CalculatorBase {
+ public:
+  LocalizationToRegionCalculator();
+  ~LocalizationToRegionCalculator() override {}
+  LocalizationToRegionCalculator(const LocalizationToRegionCalculator&) =
+      delete;
+  LocalizationToRegionCalculator& operator=(
+      const LocalizationToRegionCalculator&) = delete;
+
+  static ::mediapipe::Status GetContract(mediapipe::CalculatorContract* cc);
+  ::mediapipe::Status Open(mediapipe::CalculatorContext* cc) override;
+  ::mediapipe::Status Process(mediapipe::CalculatorContext* cc) override;
+
+ private:
+  // Calculator options.
+  LocalizationToRegionCalculatorOptions options_;
+};
+REGISTER_CALCULATOR(LocalizationToRegionCalculator);
+
+LocalizationToRegionCalculator::LocalizationToRegionCalculator() {}
+
+namespace {
+
+// Converts an object detection to a autoflip SignalType.  Returns true if the
+// std::string label has a autoflip label.
+bool MatchType(const std::string& label, SignalType* type) {
+  if (label == "person") {
+    type->set_standard(SignalType::HUMAN);
+    return true;
+  }
+  if (label == "car" || label == "truck") {
+    type->set_standard(SignalType::CAR);
+    return true;
+  }
+  if (label == "dog" || label == "cat" || label == "bird" || label == "horse") {
+    type->set_standard(SignalType::PET);
+    return true;
+  }
+  return false;
+}
+
+// Converts a detection to a SalientRegion with a given label.
+void FillSalientRegion(const mediapipe::Detection& detection,
+                       const SignalType& label, SalientRegion* region) {
+  const auto& location = detection.location_data().relative_bounding_box();
+  region->mutable_location_normalized()->set_x(location.xmin());
+  region->mutable_location_normalized()->set_y(location.ymin());
+  region->mutable_location_normalized()->set_width(location.width());
+  region->mutable_location_normalized()->set_height(location.height());
+  region->set_score(1.0);
+  *region->mutable_signal_type() = label;
+}
+
+}  // namespace
+
+::mediapipe::Status LocalizationToRegionCalculator::GetContract(
+    mediapipe::CalculatorContract* cc) {
+  cc->Inputs().Tag("DETECTIONS").Set<std::vector<mediapipe::Detection>>();
+  cc->Outputs().Tag("REGIONS").Set<DetectionSet>();
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status LocalizationToRegionCalculator::Open(
+    mediapipe::CalculatorContext* cc) {
+  options_ = cc->Options<LocalizationToRegionCalculatorOptions>();
+
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status LocalizationToRegionCalculator::Process(
+    mediapipe::CalculatorContext* cc) {
+  const auto& annotations =
+      cc->Inputs().Tag("DETECTIONS").Get<std::vector<mediapipe::Detection>>();
+  auto regions = ::absl::make_unique<DetectionSet>();
+  for (const auto& detection : annotations) {
+    RET_CHECK_EQ(detection.label().size(), 1)
+        << "Number of labels not equal to one.";
+    SignalType autoflip_label;
+    if (MatchType(detection.label(0), &autoflip_label) &&
+        options_.output_standard_signals()) {
+      FillSalientRegion(detection, autoflip_label, regions->add_detections());
+    }
+    if (options_.output_all_signals()) {
+      SignalType object;
+      object.set_standard(SignalType::OBJECT);
+      FillSalientRegion(detection, object, regions->add_detections());
+    }
+  }
+
+  cc->Outputs().Tag("REGIONS").Add(regions.release(), cc->InputTimestamp());
+  return ::mediapipe::OkStatus();
+}
+
+}  // namespace autoflip
+}  // namespace mediapipe

mediapipe/examples/desktop/autoflip/calculators/localization_to_region_calculator.proto

@@ -0,0 +1,33 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe.autoflip;
+
+import "mediapipe/framework/calculator.proto";
+
+message LocalizationToRegionCalculatorOptions {
+  extend mediapipe.CalculatorOptions {
+    optional LocalizationToRegionCalculatorOptions ext = 284226721;
+  }
+
+  // Output standard autoflip signals only (Human, Pet, Car, etc) and apply
+  // standard autoflip labels.
+  optional bool output_standard_signals = 1 [default = true];
+  // Output all signals (regardless of label) and set autoflip label as
+  // 'Object'.  Can be combined with output_standard_signals giving each
+  // detection a 'object' label and a autoflip sepcific label.
+  optional bool output_all_signals = 2 [default = false];
+}

mediapipe/examples/desktop/autoflip/calculators/localization_to_region_calculator_test.cc

@@ -0,0 +1,164 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "absl/strings/string_view.h"
+#include "mediapipe/examples/desktop/autoflip/autoflip_messages.pb.h"
+#include "mediapipe/examples/desktop/autoflip/calculators/localization_to_region_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/formats/detection.pb.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/port/status_matchers.h"
+
+using mediapipe::Detection;
+
+namespace mediapipe {
+namespace autoflip {
+namespace {
+
+const char kConfig[] = R"(
+    calculator: "LocalizationToRegionCalculator"
+    input_stream: "DETECTIONS:detections"
+    output_stream: "REGIONS:regions"
+    )";
+
+const char kCar[] = R"(
+           label: "car"
+           location_data {
+           format: RELATIVE_BOUNDING_BOX
+           relative_bounding_box {
+             xmin: -0.00375
+             ymin: 0.003333
+             width: 0.125
+             height: 0.33333
+           }
+         })";
+
+const char kDog[] = R"(
+          label: "dog"
+           location_data {
+           format: RELATIVE_BOUNDING_BOX
+           relative_bounding_box {
+             xmin: 0.0025
+             ymin: 0.005
+             width: 0.25
+             height: 0.5
+           }
+         })";
+
+const char kZebra[] = R"(
+          label: "zebra"
+          location_data {
+           format: RELATIVE_BOUNDING_BOX
+           relative_bounding_box {
+             xmin: 0.0
+             ymin: 0.0
+             width: 0.5
+             height: 0.5
+           }
+         })";
+
+void SetInputs(CalculatorRunner* runner,
+               const std::vector<std::string>& detections) {
+  auto inputs = ::absl::make_unique<std::vector<Detection>>();
+  // A face with landmarks.
+  for (const auto& detection : detections) {
+    inputs->push_back(ParseTextProtoOrDie<Detection>(detection));
+  }
+  runner->MutableInputs()
+      ->Tag("DETECTIONS")
+      .packets.push_back(Adopt(inputs.release()).At(Timestamp::PostStream()));
+}
+
+CalculatorGraphConfig::Node MakeConfig(bool output_standard, bool output_all) {
+  auto config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfig);
+
+  config.mutable_options()
+      ->MutableExtension(LocalizationToRegionCalculatorOptions::ext)
+      ->set_output_standard_signals(output_standard);
+
+  config.mutable_options()
+      ->MutableExtension(LocalizationToRegionCalculatorOptions::ext)
+      ->set_output_all_signals(output_all);
+
+  return config;
+}
+
+TEST(LocalizationToRegionCalculatorTest, StandardTypes) {
+  // Setup test
+  auto runner = ::absl::make_unique<CalculatorRunner>(MakeConfig(true, false));
+  SetInputs(runner.get(), {kCar, kDog, kZebra});
+
+  // Run the calculator.
+  MP_ASSERT_OK(runner->Run());
+
+  // Check the output regions.
+  const std::vector<Packet>& output_packets =
+      runner->Outputs().Tag("REGIONS").packets;
+  ASSERT_EQ(1, output_packets.size());
+  const auto& regions = output_packets[0].Get<DetectionSet>();
+  ASSERT_EQ(2, regions.detections().size());
+  const auto& detection = regions.detections(0);
+  EXPECT_EQ(detection.signal_type().standard(), SignalType::CAR);
+  EXPECT_FLOAT_EQ(detection.location_normalized().x(), -0.00375);
+  EXPECT_FLOAT_EQ(detection.location_normalized().y(), 0.003333);
+  EXPECT_FLOAT_EQ(detection.location_normalized().width(), 0.125);
+  EXPECT_FLOAT_EQ(detection.location_normalized().height(), 0.33333);
+  const auto& detection_1 = regions.detections(1);
+  EXPECT_EQ(detection_1.signal_type().standard(), SignalType::PET);
+  EXPECT_FLOAT_EQ(detection_1.location_normalized().x(), 0.0025);
+  EXPECT_FLOAT_EQ(detection_1.location_normalized().y(), 0.005);
+  EXPECT_FLOAT_EQ(detection_1.location_normalized().width(), 0.25);
+  EXPECT_FLOAT_EQ(detection_1.location_normalized().height(), 0.5);
+}
+
+TEST(LocalizationToRegionCalculatorTest, AllTypes) {
+  // Setup test
+  auto runner = ::absl::make_unique<CalculatorRunner>(MakeConfig(false, true));
+  SetInputs(runner.get(), {kCar, kDog, kZebra});
+
+  // Run the calculator.
+  MP_ASSERT_OK(runner->Run());
+
+  // Check the output regions.
+  const std::vector<Packet>& output_packets =
+      runner->Outputs().Tag("REGIONS").packets;
+  ASSERT_EQ(1, output_packets.size());
+  const auto& regions = output_packets[0].Get<DetectionSet>();
+  ASSERT_EQ(3, regions.detections().size());
+}
+
+TEST(LocalizationToRegionCalculatorTest, BothTypes) {
+  // Setup test
+  auto runner = ::absl::make_unique<CalculatorRunner>(MakeConfig(true, true));
+  SetInputs(runner.get(), {kCar, kDog, kZebra});
+
+  // Run the calculator.
+  MP_ASSERT_OK(runner->Run());
+
+  // Check the output regions.
+  const std::vector<Packet>& output_packets =
+      runner->Outputs().Tag("REGIONS").packets;
+  ASSERT_EQ(1, output_packets.size());
+  const auto& regions = output_packets[0].Get<DetectionSet>();
+  ASSERT_EQ(5, regions.detections().size());
+}
+
+}  // namespace
+}  // namespace autoflip
+}  // namespace mediapipe

mediapipe/examples/desktop/autoflip/calculators/scene_cropping_calculator.cc

@@ -0,0 +1,589 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/examples/desktop/autoflip/calculators/scene_cropping_calculator.h"
+
+#include <cmath>
+
+#include "absl/memory/memory.h"
+#include "absl/strings/str_format.h"
+#include "mediapipe/examples/desktop/autoflip/autoflip_messages.pb.h"
+#include "mediapipe/examples/desktop/autoflip/quality/scene_cropping_viz.h"
+#include "mediapipe/examples/desktop/autoflip/quality/utils.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/image_frame.h"
+#include "mediapipe/framework/formats/image_frame_opencv.h"
+#include "mediapipe/framework/port/opencv_core_inc.h"
+#include "mediapipe/framework/port/opencv_imgproc_inc.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/timestamp.h"
+
+namespace mediapipe {
+namespace autoflip {
+
+constexpr char kInputVideoFrames[] = "VIDEO_FRAMES";
+constexpr char kInputKeyFrames[] = "KEY_FRAMES";
+constexpr char kInputDetections[] = "DETECTION_FEATURES";
+constexpr char kInputStaticFeatures[] = "STATIC_FEATURES";
+constexpr char kInputShotBoundaries[] = "SHOT_BOUNDARIES";
+constexpr char kInputExternalSettings[] = "EXTERNAL_SETTINGS";
+// This side packet must be used in conjunction with
+// TargetSizeType::MAXIMIZE_TARGET_DIMENSION
+constexpr char kAspectRatio[] = "EXTERNAL_ASPECT_RATIO";
+
+constexpr char kOutputCroppedFrames[] = "CROPPED_FRAMES";
+constexpr char kOutputKeyFrameCropViz[] = "KEY_FRAME_CROP_REGION_VIZ_FRAMES";
+constexpr char kOutputFocusPointFrameViz[] = "SALIENT_POINT_FRAME_VIZ_FRAMES";
+constexpr char kOutputSummary[] = "CROPPING_SUMMARY";
+
+::mediapipe::Status SceneCroppingCalculator::GetContract(
+    ::mediapipe::CalculatorContract* cc) {
+  if (cc->InputSidePackets().HasTag(kInputExternalSettings)) {
+    cc->InputSidePackets().Tag(kInputExternalSettings).Set<std::string>();
+  }
+  if (cc->InputSidePackets().HasTag(kAspectRatio)) {
+    cc->InputSidePackets().Tag(kAspectRatio).Set<std::string>();
+  }
+  cc->Inputs().Tag(kInputVideoFrames).Set<ImageFrame>();
+  if (cc->Inputs().HasTag(kInputKeyFrames)) {
+    cc->Inputs().Tag(kInputKeyFrames).Set<ImageFrame>();
+  }
+  cc->Inputs().Tag(kInputDetections).Set<DetectionSet>();
+  if (cc->Inputs().HasTag(kInputStaticFeatures)) {
+    cc->Inputs().Tag(kInputStaticFeatures).Set<StaticFeatures>();
+  }
+  cc->Inputs().Tag(kInputShotBoundaries).Set<bool>();
+
+  cc->Outputs().Tag(kOutputCroppedFrames).Set<ImageFrame>();
+  if (cc->Outputs().HasTag(kOutputKeyFrameCropViz)) {
+    cc->Outputs().Tag(kOutputKeyFrameCropViz).Set<ImageFrame>();
+  }
+  if (cc->Outputs().HasTag(kOutputFocusPointFrameViz)) {
+    cc->Outputs().Tag(kOutputFocusPointFrameViz).Set<ImageFrame>();
+  }
+  if (cc->Outputs().HasTag(kOutputSummary)) {
+    cc->Outputs().Tag(kOutputSummary).Set<VideoCroppingSummary>();
+  }
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status SceneCroppingCalculator::Open(CalculatorContext* cc) {
+  options_ = cc->Options<SceneCroppingCalculatorOptions>();
+  RET_CHECK_GT(options_.max_scene_size(), 0)
+      << "Maximum scene size is non-positive.";
+  RET_CHECK_GE(options_.prior_frame_buffer_size(), 0)
+      << "Prior frame buffer size is negative.";
+
+  RET_CHECK(options_.solid_background_frames_padding_fraction() >= 0.0 &&
+            options_.solid_background_frames_padding_fraction() <= 1.0)
+      << "Solid background frames padding fraction is not in [0, 1].";
+  const auto& padding_params = options_.padding_parameters();
+  background_contrast_ = padding_params.background_contrast();
+  RET_CHECK(background_contrast_ >= 0.0 && background_contrast_ <= 1.0)
+      << "Background contrast " << background_contrast_ << " is not in [0, 1].";
+  blur_cv_size_ = padding_params.blur_cv_size();
+  RET_CHECK_GT(blur_cv_size_, 0) << "Blur cv size is non-positive.";
+  overlay_opacity_ = padding_params.overlay_opacity();
+  RET_CHECK(overlay_opacity_ >= 0.0 && overlay_opacity_ <= 1.0)
+      << "Overlay opacity " << overlay_opacity_ << " is not in [0, 1].";
+
+  scene_cropper_ = absl::make_unique<SceneCropper>();
+  if (cc->Outputs().HasTag(kOutputSummary)) {
+    summary_ = absl::make_unique<VideoCroppingSummary>();
+  }
+  return ::mediapipe::OkStatus();
+}
+
+namespace {
+::mediapipe::Status ParseAspectRatioString(
+    const std::string& aspect_ratio_string, double* aspect_ratio) {
+  std::string error_msg =
+      "Aspect ratio std::string must be in the format of 'width:height', e.g. "
+      "'1:1' or '5:4', your input was " +
+      aspect_ratio_string;
+  auto pos = aspect_ratio_string.find(":");
+  RET_CHECK(pos != std::string::npos) << error_msg;
+  double width_ratio;
+  RET_CHECK(absl::SimpleAtod(aspect_ratio_string.substr(0, pos), &width_ratio))
+      << error_msg;
+  double height_ratio;
+  RET_CHECK(absl::SimpleAtod(
+      aspect_ratio_string.substr(pos + 1, aspect_ratio_string.size()),
+      &height_ratio))
+      << error_msg;
+  *aspect_ratio = width_ratio / height_ratio;
+  return ::mediapipe::OkStatus();
+}
+}  // namespace
+
+::mediapipe::Status SceneCroppingCalculator::Process(
+    ::mediapipe::CalculatorContext* cc) {
+  // Sets frame dimension and format.
+  if (frame_width_ < 0 &&
+      !cc->Inputs().Tag(kInputVideoFrames).Value().IsEmpty()) {
+    const auto& frame = cc->Inputs().Tag(kInputVideoFrames).Get<ImageFrame>();
+    frame_width_ = frame.Width();
+    RET_CHECK_GT(frame_width_, 0) << "Input frame width is non-positive.";
+    frame_height_ = frame.Height();
+    RET_CHECK_GT(frame_height_, 0) << "Input frame height is non-positive.";
+    frame_format_ = frame.Format();
+    target_width_ = options_.target_width();
+    target_height_ = options_.target_height();
+    if (cc->InputSidePackets().HasTag(kInputExternalSettings)) {
+      auto conversion_options = ParseTextProtoOrDie<ConversionOptions>(
+          cc->InputSidePackets()
+              .Tag(kInputExternalSettings)
+              .Get<std::string>());
+      target_width_ = conversion_options.target_width();
+      target_height_ = conversion_options.target_height();
+    }
+    target_aspect_ratio_ = static_cast<double>(target_width_) / target_height_;
+    RET_CHECK_NE(options_.target_size_type(),
+                 SceneCroppingCalculatorOptions::UNKNOWN)
+        << "TargetSizeType not set properly.";
+    // Resets target size if keep original height or width.
+    if (options_.target_size_type() ==
+        SceneCroppingCalculatorOptions::KEEP_ORIGINAL_HEIGHT) {
+      target_height_ = frame_height_;
+      target_width_ = std::round(target_height_ * target_aspect_ratio_);
+    } else if (options_.target_size_type() ==
+               SceneCroppingCalculatorOptions::KEEP_ORIGINAL_WIDTH) {
+      target_width_ = frame_width_;
+      target_height_ = std::round(target_width_ / target_aspect_ratio_);
+    } else if (options_.target_size_type() ==
+               SceneCroppingCalculatorOptions::MAXIMIZE_TARGET_DIMENSION) {
+      RET_CHECK(cc->InputSidePackets().HasTag(kAspectRatio))
+          << "MAXIMIZE_TARGET_DIMENSION is set without an "
+             "external_aspect_ratio";
+      double requested_aspect_ratio;
+      MP_RETURN_IF_ERROR(ParseAspectRatioString(
+          cc->InputSidePackets().Tag(kAspectRatio).Get<std::string>(),
+          &requested_aspect_ratio));
+      const double original_aspect_ratio =
+          static_cast<double>(frame_width_) / frame_height_;
+      if (original_aspect_ratio > requested_aspect_ratio) {
+        target_height_ = frame_height_;
+        target_width_ = std::round(target_height_ * requested_aspect_ratio);
+      } else {
+        target_width_ = frame_width_;
+        target_height_ = std::round(target_width_ / requested_aspect_ratio);
+      }
+    }
+    // Makes sure that target size is even if keep original width or height.
+    if (options_.target_size_type() !=
+        SceneCroppingCalculatorOptions::USE_TARGET_DIMENSION) {
+      if (target_width_ % 2 == 1) {
+        target_width_ = std::max(2, target_width_ - 1);
+      }
+      if (target_height_ % 2 == 1) {
+        target_height_ = std::max(2, target_height_ - 1);
+      }
+      target_aspect_ratio_ =
+          static_cast<double>(target_width_) / target_height_;
+    }
+    // Set keyframe width/height for feature upscaling (overwritten by keyframe
+    // input if provided).
+    if (options_.has_video_features_width() &&
+        options_.has_video_features_height()) {
+      key_frame_width_ = options_.video_features_width();
+      key_frame_height_ = options_.video_features_height();
+    } else if (!cc->Inputs().HasTag(kInputKeyFrames)) {
+      key_frame_width_ = frame_width_;
+      key_frame_height_ = frame_height_;
+    }
+    // Check provided dimensions.
+    RET_CHECK_GT(target_width_, 0) << "Target width is non-positive.";
+    RET_CHECK_NE(target_width_ % 2, 1)
+        << "Target width cannot be odd, because encoder expects dimension "
+           "values to be even.";
+    RET_CHECK_GT(target_height_, 0) << "Target height is non-positive.";
+    RET_CHECK_NE(target_height_ % 2, 1)
+        << "Target height cannot be odd, because encoder expects dimension "
+           "values to be even.";
+  }
+
+  // Sets key frame dimension.
+  if (cc->Inputs().HasTag(kInputKeyFrames) &&
+      !cc->Inputs().Tag(kInputKeyFrames).Value().IsEmpty() &&
+      key_frame_width_ < 0) {
+    const auto& key_frame = cc->Inputs().Tag(kInputKeyFrames).Get<ImageFrame>();
+    key_frame_width_ = key_frame.Width();
+    key_frame_height_ = key_frame.Height();
+  }
+
+  // Processes a scene when shot boundary or buffer is full.
+  bool is_end_of_scene = false;
+  if (!cc->Inputs().Tag(kInputShotBoundaries).Value().IsEmpty()) {
+    is_end_of_scene = cc->Inputs().Tag(kInputShotBoundaries).Get<bool>();
+  }
+  const bool force_buffer_flush =
+      scene_frames_.size() >= options_.max_scene_size();
+  if (!scene_frames_.empty() && (is_end_of_scene || force_buffer_flush)) {
+    MP_RETURN_IF_ERROR(ProcessScene(is_end_of_scene, cc));
+  }
+
+  // Saves frame and timestamp and whether it is a key frame.
+  if (!cc->Inputs().Tag(kInputVideoFrames).Value().IsEmpty()) {
+    LOG_EVERY_N(ERROR, 10)
+        << "------------------------ (Breathing) Time(s): "
+        << cc->Inputs().Tag(kInputVideoFrames).Value().Timestamp().Seconds();
+    const auto& frame = cc->Inputs().Tag(kInputVideoFrames).Get<ImageFrame>();
+    const cv::Mat frame_mat = formats::MatView(&frame);
+    cv::Mat copy_mat;
+    frame_mat.copyTo(copy_mat);
+    scene_frames_.push_back(copy_mat);
+    scene_frame_timestamps_.push_back(cc->InputTimestamp().Value());
+    is_key_frames_.push_back(
+        !cc->Inputs().Tag(kInputDetections).Value().IsEmpty());
+  }
+
+  // Packs key frame info.
+  if (!cc->Inputs().Tag(kInputDetections).Value().IsEmpty()) {
+    const auto& detections =
+        cc->Inputs().Tag(kInputDetections).Get<DetectionSet>();
+    KeyFrameInfo key_frame_info;
+    MP_RETURN_IF_ERROR(PackKeyFrameInfo(
+        cc->InputTimestamp().Value(), detections, frame_width_, frame_height_,
+        key_frame_width_, key_frame_height_, &key_frame_info));
+    key_frame_infos_.push_back(key_frame_info);
+  }
+
+  // Buffers static features.
+  if (cc->Inputs().HasTag(kInputStaticFeatures) &&
+      !cc->Inputs().Tag(kInputStaticFeatures).Value().IsEmpty()) {
+    static_features_.push_back(
+        cc->Inputs().Tag(kInputStaticFeatures).Get<StaticFeatures>());
+    static_features_timestamps_.push_back(cc->InputTimestamp().Value());
+  }
+
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status SceneCroppingCalculator::Close(
+    ::mediapipe::CalculatorContext* cc) {
+  if (!scene_frames_.empty()) {
+    MP_RETURN_IF_ERROR(ProcessScene(/* is_end_of_scene = */ true, cc));
+  }
+  if (cc->Outputs().HasTag(kOutputSummary)) {
+    cc->Outputs()
+        .Tag(kOutputSummary)
+        .Add(summary_.release(), Timestamp::PostStream());
+  }
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status SceneCroppingCalculator::RemoveStaticBorders() {
+  int top_border_size = 0, bottom_border_size = 0;
+  MP_RETURN_IF_ERROR(ComputeSceneStaticBordersSize(
+      static_features_, &top_border_size, &bottom_border_size));
+  const double scale = static_cast<double>(frame_height_) / key_frame_height_;
+  top_border_distance_ = std::round(scale * top_border_size);
+  const int bottom_border_distance = std::round(scale * bottom_border_size);
+  effective_frame_height_ =
+      frame_height_ - top_border_distance_ - bottom_border_distance;
+
+  if (top_border_distance_ > 0 || bottom_border_distance > 0) {
+    VLOG(1) << "Remove top border " << top_border_distance_ << " bottom border "
+            << bottom_border_distance;
+    // Remove borders from frames.
+    cv::Rect roi(0, top_border_distance_, frame_width_,
+                 effective_frame_height_);
+    for (int i = 0; i < scene_frames_.size(); ++i) {
+      cv::Mat tmp;
+      scene_frames_[i](roi).copyTo(tmp);
+      scene_frames_[i] = tmp;
+    }
+    // Adjust detection bounding boxes.
+    for (int i = 0; i < key_frame_infos_.size(); ++i) {
+      DetectionSet adjusted_detections;
+      const auto& detections = key_frame_infos_[i].detections();
+      for (int j = 0; j < detections.detections_size(); ++j) {
+        const auto& detection = detections.detections(j);
+        SalientRegion adjusted_detection = detection;
+        // Clamp the box to be within the de-bordered frame.
+        if (!ClampRect(0, top_border_distance_, frame_width_,
+                       top_border_distance_ + effective_frame_height_,
+                       adjusted_detection.mutable_location())
+                 .ok()) {
+          continue;
+        }
+        // Offset the y position.
+        adjusted_detection.mutable_location()->set_y(
+            adjusted_detection.location().y() - top_border_distance_);
+        *adjusted_detections.add_detections() = adjusted_detection;
+      }
+      *key_frame_infos_[i].mutable_detections() = adjusted_detections;
+    }
+  }
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status
+SceneCroppingCalculator::InitializeFrameCropRegionComputer() {
+  key_frame_crop_options_ = options_.key_frame_crop_options();
+  MP_RETURN_IF_ERROR(
+      SetKeyFrameCropTarget(frame_width_, effective_frame_height_,
+                            target_aspect_ratio_, &key_frame_crop_options_));
+  VLOG(1) << "Target width " << key_frame_crop_options_.target_width();
+  VLOG(1) << "Target height " << key_frame_crop_options_.target_height();
+  frame_crop_region_computer_ =
+      absl::make_unique<FrameCropRegionComputer>(key_frame_crop_options_);
+  return ::mediapipe::OkStatus();
+}
+
+void SceneCroppingCalculator::FilterKeyFrameInfo() {
+  if (!options_.user_hint_override()) {
+    return;
+  }
+  std::vector<KeyFrameInfo> user_hints_only;
+  bool has_user_hints = false;
+  for (auto key_frame : key_frame_infos_) {
+    DetectionSet user_hint_only_set;
+    for (const auto& detection : key_frame.detections().detections()) {
+      if (detection.signal_type().has_standard() &&
+          detection.signal_type().standard() == SignalType::USER_HINT) {
+        *user_hint_only_set.add_detections() = detection;
+        has_user_hints = true;
+      }
+    }
+    *key_frame.mutable_detections() = user_hint_only_set;
+    user_hints_only.push_back(key_frame);
+  }
+  if (has_user_hints) {
+    key_frame_infos_ = user_hints_only;
+  }
+}
+
+::mediapipe::Status SceneCroppingCalculator::ProcessScene(
+    const bool is_end_of_scene, CalculatorContext* cc) {
+  // Removes detections under special circumstances.
+  FilterKeyFrameInfo();
+
+  // Removes any static borders.
+  MP_RETURN_IF_ERROR(RemoveStaticBorders());
+
+  // Decides if solid background color padding is possible and sets up color
+  // interpolation functions in CIELAB. Uses linear interpolation by default.
+  MP_RETURN_IF_ERROR(FindSolidBackgroundColor(
+      static_features_, static_features_timestamps_,
+      options_.solid_background_frames_padding_fraction(),
+      &has_solid_background_, &background_color_l_function_,
+      &background_color_a_function_, &background_color_b_function_));
+
+  // Computes key frame crop regions.
+  MP_RETURN_IF_ERROR(InitializeFrameCropRegionComputer());
+  const int num_key_frames = key_frame_infos_.size();
+  std::vector<KeyFrameCropResult> key_frame_crop_results(num_key_frames);
+  for (int i = 0; i < num_key_frames; ++i) {
+    MP_RETURN_IF_ERROR(frame_crop_region_computer_->ComputeFrameCropRegion(
+        key_frame_infos_[i], &key_frame_crop_results[i]));
+  }
+
+  // Analyzes scene camera motion and generates FocusPointFrames.
+  auto analyzer_options = options_.scene_camera_motion_analyzer_options();
+  analyzer_options.set_allow_sweeping(analyzer_options.allow_sweeping() &&
+                                      !has_solid_background_);
+  scene_camera_motion_analyzer_ =
+      absl::make_unique<SceneCameraMotionAnalyzer>(analyzer_options);
+  SceneKeyFrameCropSummary scene_summary;
+  std::vector<FocusPointFrame> focus_point_frames;
+  SceneCameraMotion scene_camera_motion;
+  MP_RETURN_IF_ERROR(
+      scene_camera_motion_analyzer_->AnalyzeSceneAndPopulateFocusPointFrames(
+          key_frame_infos_, key_frame_crop_options_, key_frame_crop_results,
+          frame_width_, effective_frame_height_, scene_frame_timestamps_,
+          &scene_summary, &focus_point_frames, &scene_camera_motion));
+
+  // Crops scene frames.
+  std::vector<cv::Mat> cropped_frames;
+  MP_RETURN_IF_ERROR(scene_cropper_->CropFrames(
+      scene_summary, scene_frames_, focus_point_frames,
+      prior_focus_point_frames_, &cropped_frames));
+
+  // Formats and outputs cropped frames.
+  bool apply_padding = false;
+  float vertical_fill_precent;
+  MP_RETURN_IF_ERROR(FormatAndOutputCroppedFrames(
+      cropped_frames, &apply_padding, &vertical_fill_precent, cc));
+
+  // Caches prior FocusPointFrames if this was not the end of a scene.
+  prior_focus_point_frames_.clear();
+  if (!is_end_of_scene) {
+    const int start = std::max(0, static_cast<int>(scene_frames_.size()) -
+                                      options_.prior_frame_buffer_size());
+    for (int i = start; i < num_key_frames; ++i) {
+      prior_focus_point_frames_.push_back(focus_point_frames[i]);
+    }
+  }
+
+  // Optionally outputs visualization frames.
+  MP_RETURN_IF_ERROR(OutputVizFrames(key_frame_crop_results, focus_point_frames,
+                                     scene_summary.crop_window_width(),
+                                     scene_summary.crop_window_height(), cc));
+
+  const double start_sec = Timestamp(scene_frame_timestamps_.front()).Seconds();
+  const double end_sec = Timestamp(scene_frame_timestamps_.back()).Seconds();
+  VLOG(1) << absl::StrFormat("Processed a scene from %.2f sec to %.2f sec",
+                             start_sec, end_sec);
+
+  // Optionally makes summary.
+  if (cc->Outputs().HasTag(kOutputSummary)) {
+    auto* scene_summary = summary_->add_scene_summaries();
+    scene_summary->set_start_sec(start_sec);
+    scene_summary->set_end_sec(end_sec);
+    *(scene_summary->mutable_camera_motion()) = scene_camera_motion;
+    scene_summary->set_is_end_of_scene(is_end_of_scene);
+    scene_summary->set_is_padded(apply_padding);
+  }
+
+  key_frame_infos_.clear();
+  scene_frames_.clear();
+  scene_frame_timestamps_.clear();
+  is_key_frames_.clear();
+  static_features_.clear();
+  static_features_timestamps_.clear();
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status SceneCroppingCalculator::FormatAndOutputCroppedFrames(
+    const std::vector<cv::Mat>& cropped_frames, bool* apply_padding,
+    float* vertical_fill_precent, CalculatorContext* cc) {
+  RET_CHECK(apply_padding) << "Has padding boolean is null.";
+  if (cropped_frames.empty()) {
+    return ::mediapipe::OkStatus();
+  }
+
+  // Computes scaling factor and decides if padding is needed.
+  const int crop_width = cropped_frames.front().cols;
+  const int crop_height = cropped_frames.front().rows;
+  VLOG(1) << "crop_width = " << crop_width << " crop_height = " << crop_height;
+  const double scaling =
+      std::max(static_cast<double>(target_width_) / crop_width,
+               static_cast<double>(target_height_) / crop_height);
+  int scaled_width = std::round(scaling * crop_width);
+  int scaled_height = std::round(scaling * crop_height);
+  RET_CHECK_GE(scaled_width, target_width_)
+      << "Scaled width is less than target width - something is wrong.";
+  RET_CHECK_GE(scaled_height, target_height_)
+      << "Scaled height is less than target height - something is wrong.";
+  if (scaled_width - target_width_ <= 1) scaled_width = target_width_;
+  if (scaled_height - target_height_ <= 1) scaled_height = target_height_;
+  *apply_padding =
+      scaled_width != target_width_ || scaled_height != target_height_;
+  *vertical_fill_precent = scaled_height / static_cast<float>(target_height_);
+  if (*apply_padding) {
+    padder_ = absl::make_unique<PaddingEffectGenerator>(
+        scaled_width, scaled_height, target_aspect_ratio_);
+    VLOG(1) << "Scene is padded: scaled width = " << scaled_width
+            << " target width = " << target_width_
+            << " scaled height = " << scaled_height
+            << " target height = " << target_height_;
+  }
+
+  // Resizes cropped frames, pads frames, and output frames.
+  cv::Scalar* background_color = nullptr;
+  cv::Scalar interpolated_color;
+  const int num_frames = cropped_frames.size();
+  for (int i = 0; i < num_frames; ++i) {
+    const int64 time_ms = scene_frame_timestamps_[i];
+    const Timestamp timestamp(time_ms);
+    auto scaled_frame = absl::make_unique<ImageFrame>(
+        frame_format_, scaled_width, scaled_height);
+    auto destination = formats::MatView(scaled_frame.get());
+    if (scaled_width == crop_width && scaled_height == crop_height) {
+      cropped_frames[i].copyTo(destination);
+    } else {
+      // cubic is better quality for upscaling and area is good for downscaling
+      const int interpolation_method =
+          scaling > 1 ? cv::INTER_CUBIC : cv::INTER_AREA;
+      cv::resize(cropped_frames[i], destination, destination.size(), 0, 0,
+                 interpolation_method);
+    }
+    if (*apply_padding) {
+      if (has_solid_background_) {
+        double lab[3];
+        lab[0] = background_color_l_function_.Evaluate(time_ms);
+        lab[1] = background_color_a_function_.Evaluate(time_ms);
+        lab[2] = background_color_b_function_.Evaluate(time_ms);
+        cv::Mat3f lab_mat(1, 1, cv::Vec3f(lab[0], lab[1], lab[2]));
+        cv::Mat3f rgb_mat(1, 1);
+        // Necessary scaling of the RGB values from [0, 1] to [0, 255] based on:
+        // https://docs.opencv.org/2.4/modules/imgproc/doc/miscellaneous_transformations.html#cvtcolor
+        cv::cvtColor(lab_mat, rgb_mat, cv::COLOR_Lab2RGB);
+        rgb_mat *= 255.0;
+        auto k = rgb_mat.at<cv::Vec3f>(0, 0);
+        k[0] = k[0] < 0.0 ? 0.0 : k[0] > 255.0 ? 255.0 : k[0];
+        k[1] = k[1] < 0.0 ? 0.0 : k[1] > 255.0 ? 255.0 : k[1];
+        k[2] = k[2] < 0.0 ? 0.0 : k[2] > 255.0 ? 255.0 : k[2];
+        interpolated_color =
+            cv::Scalar(std::round(k[0]), std::round(k[1]), std::round(k[2]));
+        background_color = &interpolated_color;
+      }
+      auto padded_frame = absl::make_unique<ImageFrame>();
+      MP_RETURN_IF_ERROR(padder_->Process(
+          *scaled_frame, background_contrast_,
+          std::min({blur_cv_size_, scaled_width, scaled_height}),
+          overlay_opacity_, padded_frame.get(), background_color));
+      RET_CHECK_EQ(padded_frame->Width(), target_width_)
+          << "Padded frame width is off.";
+      RET_CHECK_EQ(padded_frame->Height(), target_height_)
+          << "Padded frame height is off.";
+      cc->Outputs()
+          .Tag(kOutputCroppedFrames)
+          .Add(padded_frame.release(), timestamp);
+    } else {
+      cc->Outputs()
+          .Tag(kOutputCroppedFrames)
+          .Add(scaled_frame.release(), timestamp);
+    }
+  }
+  return ::mediapipe::OkStatus();
+}
+
+mediapipe::Status SceneCroppingCalculator::OutputVizFrames(
+    const std::vector<KeyFrameCropResult>& key_frame_crop_results,
+    const std::vector<FocusPointFrame>& focus_point_frames,
+    const int crop_window_width, const int crop_window_height,
+    CalculatorContext* cc) const {
+  if (cc->Outputs().HasTag(kOutputKeyFrameCropViz)) {
+    std::vector<std::unique_ptr<ImageFrame>> viz_frames;
+    MP_RETURN_IF_ERROR(DrawDetectionsAndCropRegions(
+        scene_frames_, is_key_frames_, key_frame_infos_, key_frame_crop_results,
+        frame_format_, &viz_frames));
+    for (int i = 0; i < scene_frames_.size(); ++i) {
+      cc->Outputs()
+          .Tag(kOutputKeyFrameCropViz)
+          .Add(viz_frames[i].release(), Timestamp(scene_frame_timestamps_[i]));
+    }
+  }
+  if (cc->Outputs().HasTag(kOutputFocusPointFrameViz)) {
+    std::vector<std::unique_ptr<ImageFrame>> viz_frames;
+    MP_RETURN_IF_ERROR(DrawFocusPointAndCropWindow(
+        scene_frames_, focus_point_frames, options_.viz_overlay_opacity(),
+        crop_window_width, crop_window_height, frame_format_, &viz_frames));
+    for (int i = 0; i < scene_frames_.size(); ++i) {
+      cc->Outputs()
+          .Tag(kOutputFocusPointFrameViz)
+          .Add(viz_frames[i].release(), Timestamp(scene_frame_timestamps_[i]));
+    }
+  }
+  return ::mediapipe::OkStatus();
+}
+
+REGISTER_CALCULATOR(SceneCroppingCalculator);
+
+}  // namespace autoflip
+}  // namespace mediapipe

mediapipe/examples/desktop/autoflip/calculators/scene_cropping_calculator.h

@@ -0,0 +1,249 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_CALCULATORS_SCENE_CROPPING_CALCULATOR_H_
+#define MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_CALCULATORS_SCENE_CROPPING_CALCULATOR_H_
+
+#include <memory>
+#include <vector>
+
+#include "mediapipe/examples/desktop/autoflip/autoflip_messages.pb.h"
+#include "mediapipe/examples/desktop/autoflip/calculators/scene_cropping_calculator.pb.h"
+#include "mediapipe/examples/desktop/autoflip/quality/cropping.pb.h"
+#include "mediapipe/examples/desktop/autoflip/quality/focus_point.pb.h"
+#include "mediapipe/examples/desktop/autoflip/quality/frame_crop_region_computer.h"
+#include "mediapipe/examples/desktop/autoflip/quality/padding_effect_generator.h"
+#include "mediapipe/examples/desktop/autoflip/quality/piecewise_linear_function.h"
+#include "mediapipe/examples/desktop/autoflip/quality/polynomial_regression_path_solver.h"
+#include "mediapipe/examples/desktop/autoflip/quality/scene_camera_motion_analyzer.h"
+#include "mediapipe/examples/desktop/autoflip/quality/scene_cropper.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/image_frame.h"
+#include "mediapipe/framework/port/opencv_core_inc.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+namespace autoflip {
+// This calculator crops video scenes to target size, which can be of any aspect
+// ratio. The calculator supports both "landscape -> portrait", and "portrait ->
+// landscape" use cases. The two use cases are automatically determined by
+// comparing the input and output frame's aspect ratios internally.
+//
+// The target (i.e. output) frame's dimension can be specified through the
+// target_width(height) fields in the options. Both this target dimension and
+// the input dimension should be even. If either keep_original_height or
+// keep_original_width is set to true, the corresponding target dimension will
+// only be used to compute the aspect ratio (as opposed to setting the actual
+// dimension) of the output. If the output frame thus computed has an odd
+// size, it will be rounded down to an even number.
+//
+// The calculator takes shot boundary signals to identify shot boundaries, and
+// crops each scene independently. The cropping decisions are made based on
+// detection features, which are a collection of focus regions detected from
+// different signals, and then fused together by a SignalFusingCalculator. To
+// add a new type of focus signals, it should be added in the input of the
+// SignalFusingCalculator, which can take an arbitrary number of input streams.
+//
+// If after attempting to cover focus regions based on the cropping decisions
+// made, the retained frame region's aspect ratio is still different from the
+// target aspect ratio, padding will be applied. In this case, a seamless
+// padding with a solid color would be preferred wherever possible, given
+// information from the input static features; otherwise, a simple padding with
+// centered foreground on blurred background will be applied.
+//
+// The main complexity of this calculator lies in stabilizing crop regions over
+// the scene using a Retargeter, which solves linear programming problems
+// through a L1 path solver (default) or least squares problems through a L2
+// path solver.
+
+// Input streams:
+// - required tag VIDEO_FRAMES (type ImageFrame):
+//     Original scene frames to be cropped.
+// - required tag DETECTION_FEATURES (type DetectionSet):
+//     Detected features on the key frames.
+// - optional tag STATIC_FEATURES (type StaticFeatures):
+//     Detected features on the key frames.
+// - required tag SHOT_BOUNDARIES (type bool):
+//     Indicators for shot boundaries (output of shot boundary detection).
+// - optional tag KEY_FRAMES (type ImageFrame):
+//     Key frames on which features are detected. This is only used to set the
+//     detection features frame size, and when it is omitted, the features frame
+//     size is assumed to be the original scene frame size.
+//
+// Output streams:
+// - required tag CROPPED_FRAMES (type ImageFrame):
+//     Cropped frames at target size and original frame rate.
+// - optional tag KEY_FRAME_CROP_REGION_VIZ_FRAMES (type ImageFrame):
+//     Debug visualization frames at original frame size and frame rate. Draws
+//     the required (yellow) and non-required (cyan) detection features and the
+//     key frame crop regions (green).
+// - optional tag SALIENT_POINT_FRAME_VIZ_FRAMES (type ImageFrame):
+//     Debug visualization frames at original frame size and frame rate. Draws
+//     the focus points and the scene crop window (red).
+// - optional tag CROPPING_SUMMARY (type VideoCroppingSummary):
+//     Debug summary information for the video. Only generates one packet when
+//     calculator closes.
+//
+// Example config:
+// node {
+//   calculator: "SceneCroppingCalculator"
+//   input_stream: "VIDEO_FRAMES:camera_frames_org"
+//   input_stream: "KEY_FRAMES:down_sampled_frames"
+//   input_stream: "DETECTION_FEATURES:focus_regions"
+//   input_stream: "STATIC_FEATURES:border_features"
+//   input_stream: "SHOT_BOUNDARIES:shot_boundary_frames"
+//   output_stream: "CROPPED_FRAMES:cropped_frames"
+//   options: {
+//     [mediapipe.SceneCroppingCalculatorOptions.ext]: {
+//       target_width: 720
+//       target_height: 1124
+//       target_size_type: USE_TARGET_DIMENSION
+//     }
+//   }
+// }
+// Note that only the target size is required in the options, and all other
+// fields are optional with default settings.
+class SceneCroppingCalculator : public CalculatorBase {
+ public:
+  static ::mediapipe::Status GetContract(CalculatorContract* cc);
+
+  // Validates calculator options and initializes SceneCameraMotionAnalyzer and
+  // SceneCropper.
+  ::mediapipe::Status Open(CalculatorContext* cc) override;
+
+  // Buffers each scene frame and its timestamp. Packs and stores KeyFrameInfo
+  // for key frames (a.k.a. frames with detection features). When a shot
+  // boundary is encountered or when the buffer is full, calls ProcessScene()
+  // to process the scene at once, and clears buffers.
+  ::mediapipe::Status Process(CalculatorContext* cc) override;
+
+  // Calls ProcessScene() on remaining buffered frames. Optionally outputs a
+  // VideoCroppingSummary if the output stream CROPPING_SUMMARY is present.
+  ::mediapipe::Status Close(::mediapipe::CalculatorContext* cc) override;
+
+ private:
+  // Removes any static borders from the scene frames before cropping.
+  ::mediapipe::Status RemoveStaticBorders();
+
+  // Initializes a FrameCropRegionComputer given input and target frame sizes.
+  ::mediapipe::Status InitializeFrameCropRegionComputer();
+
+  // Processes a scene using buffered scene frames and KeyFrameInfos:
+  // 1. Computes key frame crop regions using a FrameCropRegionComputer.
+  // 2. Analyzes scene camera motion and generates FocusPointFrames using a
+  //    SceneCameraMotionAnalyzer.
+  // 3. Crops scene frames using a SceneCropper (wrapper around Retargeter).
+  // 4. Formats and outputs cropped frames .
+  // 5. Caches prior FocusPointFrames if this is not the end of a scene (due
+  //    to force flush).
+  // 6. Optionally outputs visualization frames.
+  // 7. Optionally updates cropping summary.
+  ::mediapipe::Status ProcessScene(const bool is_end_of_scene,
+                                   CalculatorContext* cc);
+
+  // Formats and outputs the cropped frames. Scales them to be at least as big
+  // as the target size. If the aspect ratio is different, applies padding. Uses
+  // solid background from static features if possible, otherwise uses blurred
+  // background. Sets apply_padding to true if the scene is padded.
+  ::mediapipe::Status FormatAndOutputCroppedFrames(
+      const std::vector<cv::Mat>& cropped_frames, bool* apply_padding,
+      float* vertical_fill_precent, CalculatorContext* cc);
+
+  // Draws and outputs visualization frames if those streams are present.
+  ::mediapipe::Status OutputVizFrames(
+      const std::vector<KeyFrameCropResult>& key_frame_crop_results,
+      const std::vector<FocusPointFrame>& focus_point_frames,
+      const int crop_window_width, const int crop_window_height,
+      CalculatorContext* cc) const;
+
+  // Filters detections based on USER_HINT under specific flag conditions.
+  void FilterKeyFrameInfo();
+
+  // Target frame size and aspect ratio passed in or computed from options.
+  int target_width_ = -1;
+  int target_height_ = -1;
+  double target_aspect_ratio_ = -1.0;
+
+  // Input video frame size and format.
+  int frame_width_ = -1;
+  int frame_height_ = -1;
+  ImageFormat::Format frame_format_ = ImageFormat::UNKNOWN;
+
+  // Key frame size (frame size for detections and border detections).
+  int key_frame_width_ = -1;
+  int key_frame_height_ = -1;
+
+  // Calculator options.
+  SceneCroppingCalculatorOptions options_;
+
+  // Buffered KeyFrameInfos for the current scene (size = number of key frames).
+  std::vector<KeyFrameInfo> key_frame_infos_;
+
+  // Buffered frames, timestamps, and indicators for key frames in the current
+  // scene (size = number of input video frames).
+  std::vector<cv::Mat> scene_frames_;
+  std::vector<int64> scene_frame_timestamps_;
+  std::vector<bool> is_key_frames_;
+
+  // Static border information for the scene.
+  int top_border_distance_ = -1;
+  int effective_frame_height_ = -1;
+
+  // Stored FocusPointFrames from prior scene when there was no actual scene
+  // change (due to forced flush when buffer is full).
+  std::vector<FocusPointFrame> prior_focus_point_frames_;
+
+  // KeyFrameCropOptions used by the FrameCropRegionComputer.
+  KeyFrameCropOptions key_frame_crop_options_;
+
+  // Object for computing key frame crop regions from detection features.
+  std::unique_ptr<FrameCropRegionComputer> frame_crop_region_computer_ =
+      nullptr;
+
+  // Object for analyzing scene camera motion from key frame crop regions and
+  // generating FocusPointFrames.
+  std::unique_ptr<SceneCameraMotionAnalyzer> scene_camera_motion_analyzer_ =
+      nullptr;
+
+  // Object for cropping a scene given FocusPointFrames.
+  std::unique_ptr<SceneCropper> scene_cropper_ = nullptr;
+
+  // Buffered static features and their timestamps used in padding with solid
+  // background color (size = number of frames with static features).
+  std::vector<StaticFeatures> static_features_;
+  std::vector<int64> static_features_timestamps_;
+  bool has_solid_background_ = false;
+  // CIELAB yields more natural color transitions than RGB and HSV: RGB tends to
+  // produce darker in-between colors and HSV can introduce new hues. See
+  // https://howaboutanorange.com/blog/2011/08/10/color_interpolation/ for
+  // visual comparisons of color transition in different spaces.
+  PiecewiseLinearFunction background_color_l_function_;  // CIELAB - l
+  PiecewiseLinearFunction background_color_a_function_;  // CIELAB - a
+  PiecewiseLinearFunction background_color_b_function_;  // CIELAB - b
+
+  // Parameters for padding with blurred background passed in from options.
+  float background_contrast_ = -1.0;
+  int blur_cv_size_ = -1;
+  float overlay_opacity_ = -1.0;
+  // Object for padding an image to a target aspect ratio.
+  std::unique_ptr<PaddingEffectGenerator> padder_ = nullptr;
+
+  // Optional diagnostic summary output emitted in Close().
+  std::unique_ptr<VideoCroppingSummary> summary_ = nullptr;
+};
+}  // namespace autoflip
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_CALCULATORS_SCENE_CROPPING_CALCULATOR_H_

mediapipe/examples/desktop/autoflip/calculators/scene_cropping_calculator.proto

@@ -0,0 +1,101 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe.autoflip;
+
+import "mediapipe/examples/desktop/autoflip/quality/cropping.proto";
+import "mediapipe/framework/calculator.proto";
+
+// Options for the SceneCroppingCalculator.
+message SceneCroppingCalculatorOptions {
+  extend mediapipe.CalculatorOptions {
+    optional SceneCroppingCalculatorOptions ext = 284806831;
+  }
+
+  // Target frame size - this has to be even (for ffmpeg encoding).
+  optional int32 target_width = 1;
+  optional int32 target_height = 2;
+
+  // Choices for target size specification.
+  enum TargetSizeType {
+    // Unknown type (needed by ProtoBestPractices to ensure consistent behavior
+    // across proto2 and proto3). This type should not be used.
+    UNKNOWN = 0;
+    // Directly uses the target dimension given above.
+    USE_TARGET_DIMENSION = 1;
+    // Uses the target dimension to compute the target aspect ratio, but keeps
+    // original height/width. If the resulting size for the other dimension is
+    // odd, it is rounded down to an even size.
+    KEEP_ORIGINAL_HEIGHT = 2;
+    KEEP_ORIGINAL_WIDTH = 3;
+    // Used on conjuntion with external_aspect_ratio, create the largest sized
+    // output without upscaling the video.
+    MAXIMIZE_TARGET_DIMENSION = 4;
+  }
+  optional TargetSizeType target_size_type = 3 [default = USE_TARGET_DIMENSION];
+
+  // Forces a flush of the frame buffer after this number of frames even if
+  // there is not a shot boundary.
+  optional int32 max_scene_size = 4 [default = 600];
+
+  // Number of frames from prior buffer to be used to smooth out camera
+  // trajectory when it was a forced flush.
+  optional int32 prior_frame_buffer_size = 5 [default = 30];
+
+  // Options for computing key frame crop regions using the
+  // FrameCropRegionComputer.
+  // **** Note: You shall NOT manually set the target width and height fields
+  // inside this field as they will be overridden internally in the calculator
+  // (i.e. automatically computed from target aspect ratio).
+  optional KeyFrameCropOptions key_frame_crop_options = 6;
+
+  // Options for analyzing scene camera motion and populating SalientPointFrames
+  // using the SceneCameraMotionAnalyzer.
+  optional SceneCameraMotionAnalyzerOptions
+      scene_camera_motion_analyzer_options = 7;
+
+  // If the fraction of frames with solid background in one shot exceeds this
+  // threshold, use a solid color for background in padding for this shot.
+  optional float solid_background_frames_padding_fraction = 8 [default = 0.6];
+
+  // Options for padding using the PaddingEffectGenerator (copied from
+  // ad_creation/calculators/universal_padding_calculator.proto).
+  message PaddingEffectParameters {
+    // Contrast adjustment for padding background. This value should between 0
+    // and 1. The smaller the value, the darker the background. 1 means no
+    // contrast change.
+    optional float background_contrast = 1 [default = 1.0];
+    // The cv::Size() parameter used in creating blurry effects for padding
+    // backgrounds.
+    optional int32 blur_cv_size = 2 [default = 200];
+    // The opacity of the black layer overlaied on top of the background. The
+    // value should be within [0, 1], in which 0 means totally transparent, and
+    // 1 means totally opaque.
+    optional float overlay_opacity = 3 [default = 0.6];
+  }
+  optional PaddingEffectParameters padding_parameters = 9;
+
+  // If set and input "KEY_FRAMES" not provided, uses these keyframe values.
+  optional int32 video_features_width = 10;
+  optional int32 video_features_height = 11;
+
+  // If a user hint is provided on a scene, use only this signal for cropping
+  // and camera motion.
+  optional bool user_hint_override = 12;
+
+  // An opacity used to render cropping windows for visualization purposes.
+  optional float viz_overlay_opacity = 13 [default = 0.7];
+}

mediapipe/examples/desktop/autoflip/calculators/scene_cropping_calculator_test.cc

@@ -0,0 +1,621 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/examples/desktop/autoflip/calculators/scene_cropping_calculator.h"
+
+#include <random>
+#include <utility>
+#include <vector>
+
+#include "mediapipe/examples/desktop/autoflip/autoflip_messages.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/formats/image_frame_opencv.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/opencv_core_inc.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/port/status_matchers.h"
+
+namespace mediapipe {
+namespace autoflip {
+namespace {
+
+using ::testing::HasSubstr;
+
+constexpr char kConfig[] = R"(
+  calculator: "SceneCroppingCalculator"
+  input_stream: "VIDEO_FRAMES:camera_frames_org"
+  input_stream: "KEY_FRAMES:down_sampled_frames"
+  input_stream: "DETECTION_FEATURES:salient_regions"
+  input_stream: "STATIC_FEATURES:border_features"
+  input_stream: "SHOT_BOUNDARIES:shot_boundary_frames"
+  output_stream: "CROPPED_FRAMES:cropped_frames"
+  options: {
+    [mediapipe.autoflip.SceneCroppingCalculatorOptions.ext]: {
+      target_width: $0
+      target_height: $1
+      target_size_type: $2
+      max_scene_size: $3
+      prior_frame_buffer_size: $4
+    }
+  })";
+
+constexpr char kNoKeyFrameConfig[] = R"(
+  calculator: "SceneCroppingCalculator"
+  input_stream: "VIDEO_FRAMES:camera_frames_org"
+  input_stream: "DETECTION_FEATURES:salient_regions"
+  input_stream: "STATIC_FEATURES:border_features"
+  input_stream: "SHOT_BOUNDARIES:shot_boundary_frames"
+  output_stream: "CROPPED_FRAMES:cropped_frames"
+  options: {
+    [mediapipe.autoflip.SceneCroppingCalculatorOptions.ext]: {
+      target_width: $0
+      target_height: $1
+    }
+  })";
+
+constexpr char kDebugConfig[] = R"(
+  calculator: "SceneCroppingCalculator"
+  input_stream: "VIDEO_FRAMES:camera_frames_org"
+  input_stream: "KEY_FRAMES:down_sampled_frames"
+  input_stream: "DETECTION_FEATURES:salient_regions"
+  input_stream: "STATIC_FEATURES:border_features"
+  input_stream: "SHOT_BOUNDARIES:shot_boundary_frames"
+  output_stream: "CROPPED_FRAMES:cropped_frames"
+  output_stream: "KEY_FRAME_CROP_REGION_VIZ_FRAMES:key_frame_crop_viz_frames"
+  output_stream: "SALIENT_POINT_FRAME_VIZ_FRAMES:salient_point_viz_frames"
+  output_stream: "CROPPING_SUMMARY:cropping_summaries"
+  options: {
+    [mediapipe.autoflip.SceneCroppingCalculatorOptions.ext]: {
+      target_width: $0
+      target_height: $1
+    }
+  })";
+
+constexpr int kInputFrameWidth = 1280;
+constexpr int kInputFrameHeight = 720;
+
+constexpr int kKeyFrameWidth = 640;
+constexpr int kKeyFrameHeight = 360;
+
+constexpr int kTargetWidth = 720;
+constexpr int kTargetHeight = 1124;
+constexpr SceneCroppingCalculatorOptions::TargetSizeType kTargetSizeType =
+    SceneCroppingCalculatorOptions::USE_TARGET_DIMENSION;
+
+constexpr int kNumScenes = 3;
+constexpr int kSceneSize = 8;
+constexpr int kMaxSceneSize = 10;
+constexpr int kPriorFrameBufferSize = 5;
+
+constexpr int kMinNumDetections = 0;
+constexpr int kMaxNumDetections = 10;
+
+constexpr int kDownSampleRate = 4;
+constexpr int64 kTimestampDiff = 20000;
+
+// Returns a singleton random engine for generating random values. The seed is
+// fixed for reproducibility.
+std::default_random_engine& GetGen() {
+  static std::default_random_engine generator{0};
+  return generator;
+}
+
+// Returns random color with r, g, b in the range of [0, 255].
+cv::Scalar GetRandomColor() {
+  std::uniform_int_distribution<int> distribution(0, 255);
+  const int red = distribution(GetGen());
+  const int green = distribution(GetGen());
+  const int blue = distribution(GetGen());
+  return cv::Scalar(red, green, blue);
+}
+
+// Makes a detection set given number of detections. Each detection has randomly
+// generated regions within given width and height with random score in [0, 1],
+// and is randomly set to be required or non-required.
+std::unique_ptr<DetectionSet> MakeDetections(const int num_detections,
+                                             const int width,
+                                             const int height) {
+  std::uniform_int_distribution<int> width_distribution(0, width);
+  std::uniform_int_distribution<int> height_distribution(0, height);
+  std::uniform_real_distribution<float> score_distribution(0.0, 1.0);
+  std::bernoulli_distribution is_required_distribution(0.5);
+  auto detections = absl::make_unique<DetectionSet>();
+  for (int i = 0; i < num_detections; ++i) {
+    auto* region = detections->add_detections();
+    const int x1 = width_distribution(GetGen());
+    const int x2 = width_distribution(GetGen());
+    const int y1 = height_distribution(GetGen());
+    const int y2 = height_distribution(GetGen());
+    const int x_min = std::min(x1, x2), x_max = std::max(x1, x2);
+    const int y_min = std::min(y1, y2), y_max = std::max(y1, y2);
+    auto* location = region->mutable_location();
+    location->set_x(x_min);
+    location->set_width(x_max - x_min);
+    location->set_y(y_min);
+    location->set_height(y_max - y_min);
+    region->set_score(score_distribution(GetGen()));
+    region->set_is_required(is_required_distribution(GetGen()));
+  }
+  return detections;
+}
+
+// Makes an image frame of solid color given color, width, and height.
+std::unique_ptr<ImageFrame> MakeImageFrameFromColor(const cv::Scalar& color,
+                                                    const int width,
+                                                    const int height) {
+  auto image_frame =
+      absl::make_unique<ImageFrame>(ImageFormat::SRGB, width, height);
+  auto mat = formats::MatView(image_frame.get());
+  mat = color;
+  return image_frame;
+}
+
+// Adds key frame detection features given time (in ms) to the input stream.
+// Randomly generates a number of detections in the range of kMinNumDetections
+// and kMaxNumDetections. Optionally add a key image frame of random solid color
+// and given size.
+void AddKeyFrameFeatures(const int64 time_ms, const int key_frame_width,
+                         const int key_frame_height,
+                         CalculatorRunner::StreamContentsSet* inputs) {
+  Timestamp timestamp(time_ms);
+  if (inputs->HasTag("KEY_FRAMES")) {
+    auto key_frame = MakeImageFrameFromColor(GetRandomColor(), key_frame_width,
+                                             key_frame_height);
+    inputs->Tag("KEY_FRAMES")
+        .packets.push_back(Adopt(key_frame.release()).At(timestamp));
+  }
+
+  const int num_detections = std::uniform_int_distribution<int>(
+      kMinNumDetections, kMaxNumDetections)(GetGen());
+  auto detections =
+      MakeDetections(num_detections, key_frame_width, key_frame_height);
+  inputs->Tag("DETECTION_FEATURES")
+      .packets.push_back(Adopt(detections.release()).At(timestamp));
+}
+
+// Adds a scene given number of frames to the input stream. Spaces frame at the
+// default timestamp interval starting from given start frame index. Scene has
+// empty static features.
+void AddScene(const int start_frame_index, const int num_scene_frames,
+              const int frame_width, const int frame_height,
+              const int key_frame_width, const int key_frame_height,
+              CalculatorRunner::StreamContentsSet* inputs) {
+  int64 time_ms = start_frame_index * kTimestampDiff;
+  for (int i = 0; i < num_scene_frames; ++i) {
+    Timestamp timestamp(time_ms);
+    auto frame =
+        MakeImageFrameFromColor(GetRandomColor(), frame_width, frame_height);
+    inputs->Tag("VIDEO_FRAMES")
+        .packets.push_back(Adopt(frame.release()).At(timestamp));
+    auto static_features = absl::make_unique<StaticFeatures>();
+    inputs->Tag("STATIC_FEATURES")
+        .packets.push_back(Adopt(static_features.release()).At(timestamp));
+    if (i % kDownSampleRate == 0) {  // is a key frame
+      AddKeyFrameFeatures(time_ms, key_frame_width, key_frame_height, inputs);
+    }
+    if (i == num_scene_frames - 1) {  // adds shot boundary
+      inputs->Tag("SHOT_BOUNDARIES")
+          .packets.push_back(Adopt(new bool(true)).At(Timestamp(time_ms)));
+    }
+    time_ms += kTimestampDiff;
+  }
+}
+
+// Checks that the output stream for cropped frames has the correct number of
+// frames, and that the size of each frame is correct.
+void CheckCroppedFrames(const CalculatorRunner& runner, const int num_frames,
+                        const int target_width, const int target_height) {
+  const auto& outputs = runner.Outputs();
+  EXPECT_TRUE(outputs.HasTag("CROPPED_FRAMES"));
+  const auto& cropped_frames_outputs = outputs.Tag("CROPPED_FRAMES").packets;
+  EXPECT_EQ(cropped_frames_outputs.size(), num_frames);
+  for (int i = 0; i < num_frames; ++i) {
+    const auto& cropped_frame = cropped_frames_outputs[i].Get<ImageFrame>();
+    EXPECT_EQ(cropped_frame.Width(), target_width);
+    EXPECT_EQ(cropped_frame.Height(), target_height);
+  }
+}
+
+// Checks that the calculator checks the maximum scene size is valid.
+TEST(SceneCroppingCalculatorTest, ChecksMaxSceneSize) {
+  const CalculatorGraphConfig::Node config =
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(
+          absl::Substitute(kConfig, kTargetWidth, kTargetHeight,
+                           kTargetSizeType, 0, kPriorFrameBufferSize));
+  auto runner = absl::make_unique<CalculatorRunner>(config);
+  const auto status = runner->Run();
+  EXPECT_FALSE(status.ok());
+  EXPECT_THAT(status.ToString(),
+              HasSubstr("Maximum scene size is non-positive."));
+}
+
+// Checks that the calculator checks the prior frame buffer size is valid.
+TEST(SceneCroppingCalculatorTest, ChecksPriorFrameBufferSize) {
+  const CalculatorGraphConfig::Node config =
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(
+          absl::Substitute(kConfig, kTargetWidth, kTargetHeight,
+                           kTargetSizeType, kMaxSceneSize, -1));
+  auto runner = absl::make_unique<CalculatorRunner>(config);
+  const auto status = runner->Run();
+  EXPECT_FALSE(status.ok());
+  EXPECT_THAT(status.ToString(),
+              HasSubstr("Prior frame buffer size is negative."));
+}
+
+// Checks that the calculator crops scene frames when there is no input key
+// frames stream.
+TEST(SceneCroppingCalculatorTest, HandlesNoKeyFrames) {
+  const CalculatorGraphConfig::Node config =
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(
+          absl::Substitute(kNoKeyFrameConfig, kTargetWidth, kTargetHeight));
+  auto runner = absl::make_unique<CalculatorRunner>(config);
+  AddScene(0, kSceneSize, kInputFrameWidth, kInputFrameHeight, kKeyFrameWidth,
+           kKeyFrameHeight, runner->MutableInputs());
+  MP_EXPECT_OK(runner->Run());
+  CheckCroppedFrames(*runner, kSceneSize, kTargetWidth, kTargetHeight);
+}
+
+// Checks that the calculator handles scenes longer than maximum scene size (
+// force flush is triggered).
+TEST(SceneCroppingCalculatorTest, HandlesLongScene) {
+  const CalculatorGraphConfig::Node config =
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(absl::Substitute(
+          kConfig, kTargetWidth, kTargetHeight, kTargetSizeType, kMaxSceneSize,
+          kPriorFrameBufferSize));
+  auto runner = absl::make_unique<CalculatorRunner>(config);
+  AddScene(0, 2 * kMaxSceneSize, kInputFrameWidth, kInputFrameHeight,
+           kKeyFrameWidth, kKeyFrameHeight, runner->MutableInputs());
+  MP_EXPECT_OK(runner->Run());
+  CheckCroppedFrames(*runner, 2 * kMaxSceneSize, kTargetWidth, kTargetHeight);
+}
+
+// Checks that the calculator can optionally output debug streams.
+TEST(SceneCroppingCalculatorTest, OutputsDebugStreams) {
+  const CalculatorGraphConfig::Node config =
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(
+          absl::Substitute(kDebugConfig, kTargetWidth, kTargetHeight));
+  auto runner = absl::make_unique<CalculatorRunner>(config);
+  const int num_frames = kSceneSize;
+  AddScene(0, num_frames, kInputFrameWidth, kInputFrameHeight, kKeyFrameWidth,
+           kKeyFrameHeight, runner->MutableInputs());
+
+  MP_EXPECT_OK(runner->Run());
+  const auto& outputs = runner->Outputs();
+  EXPECT_TRUE(outputs.HasTag("KEY_FRAME_CROP_REGION_VIZ_FRAMES"));
+  EXPECT_TRUE(outputs.HasTag("SALIENT_POINT_FRAME_VIZ_FRAMES"));
+  EXPECT_TRUE(outputs.HasTag("CROPPING_SUMMARY"));
+  const auto& crop_region_viz_frames_outputs =
+      outputs.Tag("KEY_FRAME_CROP_REGION_VIZ_FRAMES").packets;
+  const auto& salient_point_viz_frames_outputs =
+      outputs.Tag("SALIENT_POINT_FRAME_VIZ_FRAMES").packets;
+  const auto& summary_output = outputs.Tag("CROPPING_SUMMARY").packets;
+  EXPECT_EQ(crop_region_viz_frames_outputs.size(), num_frames);
+  EXPECT_EQ(salient_point_viz_frames_outputs.size(), num_frames);
+  EXPECT_EQ(summary_output.size(), 1);
+
+  for (int i = 0; i < num_frames; ++i) {
+    const auto& crop_region_viz_frame =
+        crop_region_viz_frames_outputs[i].Get<ImageFrame>();
+    EXPECT_EQ(crop_region_viz_frame.Width(), kInputFrameWidth);
+    EXPECT_EQ(crop_region_viz_frame.Height(), kInputFrameHeight);
+    const auto& salient_point_viz_frame =
+        salient_point_viz_frames_outputs[i].Get<ImageFrame>();
+    EXPECT_EQ(salient_point_viz_frame.Width(), kInputFrameWidth);
+    EXPECT_EQ(salient_point_viz_frame.Height(), kInputFrameHeight);
+  }
+  const auto& summary = summary_output[0].Get<VideoCroppingSummary>();
+  EXPECT_EQ(summary.scene_summaries_size(), 2);
+  const auto& summary_0 = summary.scene_summaries(0);
+  EXPECT_TRUE(summary_0.is_padded());
+  EXPECT_TRUE(summary_0.camera_motion().has_steady_motion());
+}
+
+// Checks that the calculator handles the case of generating landscape frames.
+TEST(SceneCroppingCalculatorTest, HandlesLandscapeTarget) {
+  const int input_width = 900;
+  const int input_height = 1600;
+  const int target_width = 1200;
+  const int target_height = 800;
+  const CalculatorGraphConfig::Node config =
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(absl::Substitute(
+          kConfig, target_width, target_height, kTargetSizeType, kMaxSceneSize,
+          kPriorFrameBufferSize));
+  auto runner = absl::make_unique<CalculatorRunner>(config);
+  for (int i = 0; i < kNumScenes; ++i) {
+    AddScene(i * kSceneSize, kSceneSize, input_width, input_height,
+             kKeyFrameWidth, kKeyFrameHeight, runner->MutableInputs());
+  }
+  const int num_frames = kSceneSize * kNumScenes;
+  MP_EXPECT_OK(runner->Run());
+  CheckCroppedFrames(*runner, num_frames, target_width, target_height);
+}
+
+// Checks that the calculator crops scene frames to target size when the target
+// size type is the default USE_TARGET_DIMENSION.
+TEST(SceneCroppingCalculatorTest, CropsToTargetSize) {
+  const CalculatorGraphConfig::Node config =
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(absl::Substitute(
+          kConfig, kTargetWidth, kTargetHeight, kTargetSizeType, kMaxSceneSize,
+          kPriorFrameBufferSize));
+  auto runner = absl::make_unique<CalculatorRunner>(config);
+  for (int i = 0; i < kNumScenes; ++i) {
+    AddScene(i * kSceneSize, kSceneSize, kInputFrameWidth, kInputFrameHeight,
+             kKeyFrameWidth, kKeyFrameHeight, runner->MutableInputs());
+  }
+  const int num_frames = kSceneSize * kNumScenes;
+  MP_EXPECT_OK(runner->Run());
+  CheckCroppedFrames(*runner, num_frames, kTargetWidth, kTargetHeight);
+}
+
+// Checks that the calculator keeps original height if the target size type is
+// set to KEEP_ORIGINAL_HEIGHT.
+TEST(SceneCroppingCalculatorTest, KeepsOriginalHeight) {
+  const auto target_size_type =
+      SceneCroppingCalculatorOptions::KEEP_ORIGINAL_HEIGHT;
+  const int target_height = kInputFrameHeight;
+  const double target_aspect_ratio =
+      static_cast<double>(kTargetWidth) / kTargetHeight;
+  int target_width = std::round(target_height * target_aspect_ratio);
+  if (target_width % 2 == 1) target_width--;
+  const CalculatorGraphConfig::Node config =
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(absl::Substitute(
+          kConfig, kTargetWidth, kTargetHeight, target_size_type, kMaxSceneSize,
+          kPriorFrameBufferSize));
+  auto runner = absl::make_unique<CalculatorRunner>(config);
+  AddScene(0, kMaxSceneSize, kInputFrameWidth, kInputFrameHeight,
+           kKeyFrameWidth, kKeyFrameHeight, runner->MutableInputs());
+  MP_EXPECT_OK(runner->Run());
+  CheckCroppedFrames(*runner, kMaxSceneSize, target_width, target_height);
+}
+
+// Checks that the calculator keeps original width if the target size type is
+// set to KEEP_ORIGINAL_WIDTH.
+TEST(SceneCroppingCalculatorTest, KeepsOriginalWidth) {
+  const auto target_size_type =
+      SceneCroppingCalculatorOptions::KEEP_ORIGINAL_WIDTH;
+  const int target_width = kInputFrameWidth;
+  const double target_aspect_ratio =
+      static_cast<double>(kTargetWidth) / kTargetHeight;
+  int target_height = std::round(target_width / target_aspect_ratio);
+  if (target_height % 2 == 1) target_height--;
+  const CalculatorGraphConfig::Node config =
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(absl::Substitute(
+          kConfig, kTargetWidth, kTargetHeight, target_size_type, kMaxSceneSize,
+          kPriorFrameBufferSize));
+  auto runner = absl::make_unique<CalculatorRunner>(config);
+  AddScene(0, kMaxSceneSize, kInputFrameWidth, kInputFrameHeight,
+           kKeyFrameWidth, kKeyFrameHeight, runner->MutableInputs());
+  MP_EXPECT_OK(runner->Run());
+  CheckCroppedFrames(*runner, kMaxSceneSize, target_width, target_height);
+}
+
+// Checks that the calculator rejects odd target size.
+TEST(SceneCroppingCalculatorTest, RejectsOddTargetSize) {
+  const CalculatorGraphConfig::Node config =
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(absl::Substitute(
+          kConfig, kTargetWidth - 1, kTargetHeight, kTargetSizeType,
+          kMaxSceneSize, kPriorFrameBufferSize));
+  auto runner = absl::make_unique<CalculatorRunner>(config);
+  AddScene(0, kMaxSceneSize, kInputFrameWidth, kInputFrameHeight,
+           kKeyFrameWidth, kKeyFrameHeight, runner->MutableInputs());
+  const auto status = runner->Run();
+  EXPECT_FALSE(status.ok());
+  EXPECT_THAT(status.ToString(), HasSubstr("Target width cannot be odd"));
+}
+
+// Checks that the calculator always produces even frame size given even input
+// frame size and even target under all target size types.
+TEST(SceneCroppingCalculatorTest, ProducesEvenFrameSize) {
+  // Some commonly used video resolution (some are divided by 10 to make the
+  // test faster), and some odd input frame sizes.
+  const std::vector<std::pair<int, int>> video_sizes = {
+      {384, 216}, {256, 144}, {192, 108}, {128, 72},  {640, 360},
+      {426, 240}, {100, 100}, {214, 100}, {240, 100}, {720, 1124},
+      {90, 160},  {641, 360}, {640, 361}, {101, 101}};
+
+  const std::vector<SceneCroppingCalculatorOptions::TargetSizeType>
+      target_size_types = {SceneCroppingCalculatorOptions::USE_TARGET_DIMENSION,
+                           SceneCroppingCalculatorOptions::KEEP_ORIGINAL_HEIGHT,
+                           SceneCroppingCalculatorOptions::KEEP_ORIGINAL_WIDTH};
+
+  // Exhaustive check on each size as input and each size as output for each
+  // target size type.
+  for (int i = 0; i < video_sizes.size(); ++i) {
+    const int frame_width = video_sizes[i].first;
+    const int frame_height = video_sizes[i].second;
+    for (int j = 0; j < video_sizes.size(); ++j) {
+      const int target_width = video_sizes[j].first;
+      const int target_height = video_sizes[j].second;
+      if (target_width % 2 == 1 || target_height % 2 == 1) continue;
+      for (int k = 0; k < target_size_types.size(); ++k) {
+        const CalculatorGraphConfig::Node config =
+            ParseTextProtoOrDie<CalculatorGraphConfig::Node>(absl::Substitute(
+                kConfig, target_width, target_height, target_size_types[k],
+                kMaxSceneSize, kPriorFrameBufferSize));
+        auto runner = absl::make_unique<CalculatorRunner>(config);
+        AddScene(0, 1, frame_width, frame_height, kKeyFrameWidth,
+                 kKeyFrameHeight, runner->MutableInputs());
+        MP_EXPECT_OK(runner->Run());
+        const auto& output_frame = runner->Outputs()
+                                       .Tag("CROPPED_FRAMES")
+                                       .packets[0]
+                                       .Get<ImageFrame>();
+        EXPECT_EQ(output_frame.Width() % 2, 0);
+        EXPECT_EQ(output_frame.Height() % 2, 0);
+        if (target_size_types[k] ==
+            SceneCroppingCalculatorOptions::USE_TARGET_DIMENSION) {
+          EXPECT_EQ(output_frame.Width(), target_width);
+          EXPECT_EQ(output_frame.Height(), target_height);
+        } else if (target_size_types[k] ==
+                   SceneCroppingCalculatorOptions::KEEP_ORIGINAL_HEIGHT) {
+          // Difference could be 1 if input size is odd.
+          EXPECT_LE(std::abs(output_frame.Height() - frame_height), 1);
+        } else if (target_size_types[k] ==
+                   SceneCroppingCalculatorOptions::KEEP_ORIGINAL_WIDTH) {
+          EXPECT_LE(std::abs(output_frame.Width() - frame_width), 1);
+        }
+      }
+    }
+  }
+}
+
+// Checks that the calculator pads the frames with solid color when possible.
+TEST(SceneCroppingCalculatorTest, PadsWithSolidColorFromStaticFeatures) {
+  const int target_width = 100, target_height = 200;
+  const int input_width = 100, input_height = 100;
+  CalculatorGraphConfig::Node config =
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(
+          absl::Substitute(kNoKeyFrameConfig, target_width, target_height));
+  auto* options = config.mutable_options()->MutableExtension(
+      SceneCroppingCalculatorOptions::ext);
+  options->set_solid_background_frames_padding_fraction(0.6);
+  auto runner = absl::make_unique<CalculatorRunner>(config);
+
+  const int static_features_downsample_rate = 2;
+  const float fraction_with_solid_background = 0.7;
+  const int red = 122, green = 167, blue = 250;
+  const int num_frames_with_solid_background =
+      std::round(fraction_with_solid_background * kSceneSize /
+                 static_features_downsample_rate);
+
+  // Add inputs.
+  auto* inputs = runner->MutableInputs();
+  int64 time_ms = 0;
+  int num_static_features = 0;
+  for (int i = 0; i < kSceneSize; ++i) {
+    Timestamp timestamp(time_ms);
+    auto frame =
+        MakeImageFrameFromColor(GetRandomColor(), input_width, input_height);
+    inputs->Tag("VIDEO_FRAMES")
+        .packets.push_back(Adopt(frame.release()).At(timestamp));
+    if (i % static_features_downsample_rate == 0) {
+      auto static_features = absl::make_unique<StaticFeatures>();
+      if (num_static_features < num_frames_with_solid_background) {
+        auto* color = static_features->mutable_solid_background();
+        // Uses BGR to mimic input from static features solid background color.
+        color->set_r(blue);
+        color->set_g(green);
+        color->set_b(red);
+      }
+      inputs->Tag("STATIC_FEATURES")
+          .packets.push_back(Adopt(static_features.release()).At(timestamp));
+      num_static_features++;
+    }
+    if (i % kDownSampleRate == 0) {  // is a key frame
+      // Target crop size is (50, 100). Adds one required detection with size
+      // (80, 100) larger than the target crop size to force padding.
+      auto detections = absl::make_unique<DetectionSet>();
+      auto* salient_region = detections->add_detections();
+      salient_region->set_is_required(true);
+      auto* location = salient_region->mutable_location();
+      location->set_x(10);
+      location->set_y(0);
+      location->set_width(80);
+      location->set_height(input_height);
+      inputs->Tag("DETECTION_FEATURES")
+          .packets.push_back(Adopt(detections.release()).At(timestamp));
+    }
+    time_ms += kTimestampDiff;
+  }
+
+  MP_EXPECT_OK(runner->Run());
+
+  // Checks that the top and bottom borders indeed have the background color.
+  const int border_size = 37;
+  const auto& cropped_frames_outputs =
+      runner->Outputs().Tag("CROPPED_FRAMES").packets;
+  EXPECT_EQ(cropped_frames_outputs.size(), kSceneSize);
+  for (int i = 0; i < kSceneSize; ++i) {
+    const auto& cropped_frame = cropped_frames_outputs[i].Get<ImageFrame>();
+    cv::Mat mat = formats::MatView(&cropped_frame);
+    for (int x = 0; x < target_width; ++x) {
+      for (int y = 0; y < border_size; ++y) {
+        EXPECT_EQ(mat.at<cv::Vec3b>(y, x)[0], red);
+        EXPECT_EQ(mat.at<cv::Vec3b>(y, x)[1], green);
+        EXPECT_EQ(mat.at<cv::Vec3b>(y, x)[2], blue);
+      }
+      for (int y2 = 0; y2 < border_size; ++y2) {
+        const int y = target_height - 1 - y2;
+        EXPECT_EQ(mat.at<cv::Vec3b>(y, x)[0], red);
+        EXPECT_EQ(mat.at<cv::Vec3b>(y, x)[1], green);
+        EXPECT_EQ(mat.at<cv::Vec3b>(y, x)[2], blue);
+      }
+    }
+  }
+}
+
+// Checks that the calculator removes static borders from frames.
+TEST(SceneCroppingCalculatorTest, RemovesStaticBorders) {
+  const int target_width = 50, target_height = 100;
+  const int input_width = 100, input_height = 100;
+  const int top_border_size = 20, bottom_border_size = 20;
+  const cv::Rect top_border_rect(0, 0, input_width, top_border_size);
+  const cv::Rect bottom_border_rect(0, input_height - bottom_border_size,
+                                    input_width, bottom_border_size);
+  const cv::Scalar frame_color = cv::Scalar(255, 255, 255);
+  const cv::Scalar border_color = cv::Scalar(0, 0, 0);
+
+  const auto config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(
+      absl::Substitute(kNoKeyFrameConfig, target_width, target_height));
+  auto runner = absl::make_unique<CalculatorRunner>(config);
+
+  // Add inputs.
+  auto* inputs = runner->MutableInputs();
+  const auto timestamp = Timestamp(0);
+  // Make frame with borders.
+  auto frame = MakeImageFrameFromColor(frame_color, input_width, input_height);
+  auto mat = formats::MatView(frame.get());
+  mat(top_border_rect) = border_color;
+  mat(bottom_border_rect) = border_color;
+  inputs->Tag("VIDEO_FRAMES")
+      .packets.push_back(Adopt(frame.release()).At(timestamp));
+  // Set borders in static features.
+  auto static_features = absl::make_unique<StaticFeatures>();
+  auto* top_part = static_features->add_border();
+  top_part->set_relative_position(Border::TOP);
+  top_part->mutable_border_position()->set_height(top_border_size);
+  auto* bottom_part = static_features->add_border();
+  bottom_part->set_relative_position(Border::BOTTOM);
+  bottom_part->mutable_border_position()->set_height(bottom_border_size);
+  inputs->Tag("STATIC_FEATURES")
+      .packets.push_back(Adopt(static_features.release()).At(timestamp));
+  // Add empty detections to ensure no padding is used.
+  auto detections = absl::make_unique<DetectionSet>();
+  inputs->Tag("DETECTION_FEATURES")
+      .packets.push_back(Adopt(detections.release()).At(timestamp));
+
+  MP_EXPECT_OK(runner->Run());
+
+  // Checks that the top and bottom borders are removed. Each frame should have
+  // solid color equal to frame color.
+  const auto& cropped_frames_outputs =
+      runner->Outputs().Tag("CROPPED_FRAMES").packets;
+  EXPECT_EQ(cropped_frames_outputs.size(), 1);
+  const auto& cropped_frame = cropped_frames_outputs[0].Get<ImageFrame>();
+  const auto cropped_mat = formats::MatView(&cropped_frame);
+  for (int x = 0; x < target_width; ++x) {
+    for (int y = 0; y < target_height; ++y) {
+      EXPECT_EQ(cropped_mat.at<cv::Vec3b>(y, x)[0], frame_color[0]);
+      EXPECT_EQ(cropped_mat.at<cv::Vec3b>(y, x)[1], frame_color[1]);
+      EXPECT_EQ(cropped_mat.at<cv::Vec3b>(y, x)[2], frame_color[2]);
+    }
+  }
+}
+}  // namespace
+}  // namespace autoflip
+}  // namespace mediapipe

mediapipe/examples/desktop/autoflip/calculators/shot_boundary_calculator.cc

@@ -0,0 +1,190 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <map>
+#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "mediapipe/examples/desktop/autoflip/calculators/shot_boundary_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/image_frame.h"
+#include "mediapipe/framework/formats/image_frame_opencv.h"
+#include "mediapipe/framework/port/opencv_imgproc_inc.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/timestamp.h"
+
+using mediapipe::ImageFrame;
+using mediapipe::PacketTypeSet;
+
+// IO labels.
+constexpr char kVideoInputTag[] = "VIDEO";
+constexpr char kShotChangeTag[] = "IS_SHOT_CHANGE";
+// Histogram settings.
+const int kSaturationBins = 8;
+const int kHistogramChannels[] = {0, 1, 2};
+const int kHistogramBinNum[] = {kSaturationBins, kSaturationBins,
+                                kSaturationBins};
+const float kRange[] = {0, 256};
+const float* kHistogramRange[] = {kRange, kRange, kRange};
+
+namespace mediapipe {
+namespace autoflip {
+
+// This calculator computes a shot (or scene) change within a video.  It works
+// by computing a 3d color histogram and comparing this frame-to-frame. Settings
+// to control the shot change logic are presented in the options proto.
+//
+// Example:
+//  node {
+//    calculator: "ShotBoundaryCalculator"
+//    input_stream: "VIDEO:camera_frames"
+//    output_stream: "IS_SHOT_CHANGE:is_shot"
+//  }
+class ShotBoundaryCalculator : public mediapipe::CalculatorBase {
+ public:
+  ShotBoundaryCalculator() {}
+  ShotBoundaryCalculator(const ShotBoundaryCalculator&) = delete;
+  ShotBoundaryCalculator& operator=(const ShotBoundaryCalculator&) = delete;
+
+  static ::mediapipe::Status GetContract(mediapipe::CalculatorContract* cc);
+  mediapipe::Status Open(mediapipe::CalculatorContext* cc) override;
+  mediapipe::Status Process(mediapipe::CalculatorContext* cc) override;
+
+ private:
+  // Computes the histogram of an image.
+  void ComputeHistogram(const cv::Mat& image, cv::Mat* image_histogram);
+  // Transmits signal to next calculator.
+  void Transmit(mediapipe::CalculatorContext* cc, bool is_shot_change);
+  // Calculator options.
+  ShotBoundaryCalculatorOptions options_;
+  // Last time a shot was detected.
+  Timestamp last_shot_timestamp_;
+  // Defines if the calculator has received a frame yet.
+  bool init_;
+  // Histogram from the last frame.
+  cv::Mat last_histogram_;
+  // History of histogram motion.
+  std::deque<double> motion_history_;
+};
+REGISTER_CALCULATOR(ShotBoundaryCalculator);
+
+void ShotBoundaryCalculator::ComputeHistogram(const cv::Mat& image,
+                                              cv::Mat* image_histogram) {
+  cv::Mat equalized_image;
+  cv::cvtColor(image.clone(), equalized_image, CV_RGB2GRAY);
+
+  double min, max;
+  cv::minMaxLoc(equalized_image, &min, &max);
+
+  if (options_.equalize_histogram()) {
+    cv::equalizeHist(equalized_image, equalized_image);
+  }
+
+  cv::calcHist(&image, 1, kHistogramChannels, cv::Mat(), *image_histogram, 2,
+               kHistogramBinNum, kHistogramRange, true, false);
+}
+
+mediapipe::Status ShotBoundaryCalculator::Open(
+    mediapipe::CalculatorContext* cc) {
+  options_ = cc->Options<ShotBoundaryCalculatorOptions>();
+  last_shot_timestamp_ = Timestamp(0);
+  init_ = false;
+  return ::mediapipe::OkStatus();
+}
+
+void ShotBoundaryCalculator::Transmit(mediapipe::CalculatorContext* cc,
+                                      bool is_shot_change) {
+  if ((cc->InputTimestamp() - last_shot_timestamp_).Seconds() <
+      options_.min_shot_span()) {
+    is_shot_change = false;
+  }
+  if (is_shot_change) {
+    LOG(INFO) << "Shot change at: " << cc->InputTimestamp().Seconds()
+              << " seconds.";
+    cc->Outputs()
+        .Tag(kShotChangeTag)
+        .AddPacket(Adopt(std::make_unique<bool>(true).release())
+                       .At(cc->InputTimestamp()));
+  } else if (!options_.output_only_on_change()) {
+    cc->Outputs()
+        .Tag(kShotChangeTag)
+        .AddPacket(Adopt(std::make_unique<bool>(false).release())
+                       .At(cc->InputTimestamp()));
+  }
+}
+
+::mediapipe::Status ShotBoundaryCalculator::Process(
+    mediapipe::CalculatorContext* cc) {
+  // Connect to input frame and make a mutable copy.
+  cv::Mat frame_org = mediapipe::formats::MatView(
+      &cc->Inputs().Tag(kVideoInputTag).Get<ImageFrame>());
+  cv::Mat frame = frame_org.clone();
+
+  // Extract histogram from the current frame.
+  cv::Mat current_histogram;
+  ComputeHistogram(frame, &current_histogram);
+
+  if (!init_) {
+    last_histogram_ = current_histogram;
+    init_ = true;
+    Transmit(cc, false);
+    return ::mediapipe::OkStatus();
+  }
+
+  double current_motion_estimate =
+      1 - cv::compareHist(current_histogram, last_histogram_, CV_COMP_CORREL);
+  last_histogram_ = current_histogram;
+  motion_history_.push_front(current_motion_estimate);
+
+  if (motion_history_.size() != options_.window_size()) {
+    Transmit(cc, false);
+    return ::mediapipe::OkStatus();
+  }
+
+  // Shot detection algorithm is a mixture of adaptive (controlled with
+  // shot_measure) and hard thresholds. In saturation it uses hard thresholds
+  // to account for black startups, shot cuts across high motion etc.
+  // In the operating region it uses an adaptive threshold to tune motion vs.
+  // cut boundary.
+  double current_max =
+      *std::max_element(motion_history_.begin(), motion_history_.end());
+  double shot_measure = current_motion_estimate / current_max;
+
+  if ((shot_measure > options_.min_shot_measure() &&
+       current_motion_estimate > options_.min_motion_with_shot_measure()) ||
+      current_motion_estimate > options_.min_motion()) {
+    Transmit(cc, true);
+    last_shot_timestamp_ = cc->InputTimestamp();
+  } else {
+    Transmit(cc, false);
+  }
+
+  // Store histogram for next frame.
+  last_histogram_ = current_histogram;
+  motion_history_.pop_back();
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status ShotBoundaryCalculator::GetContract(
+    mediapipe::CalculatorContract* cc) {
+  cc->Inputs().Tag(kVideoInputTag).Set<ImageFrame>();
+  cc->Outputs().Tag(kShotChangeTag).Set<bool>();
+  return ::mediapipe::OkStatus();
+}
+
+}  // namespace autoflip
+}  // namespace mediapipe

mediapipe/examples/desktop/autoflip/calculators/shot_boundary_calculator.proto

@@ -0,0 +1,48 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe.autoflip;
+
+import "mediapipe/framework/calculator.proto";
+
+message ShotBoundaryCalculatorOptions {
+  extend mediapipe.CalculatorOptions {
+    optional ShotBoundaryCalculatorOptions ext = 281194049;
+  }
+  // Parameters to shot detection algorithm. All the constraints (the fields
+  // named with 'min_') need to be satisfied for a frame to be a shot boundary.
+  //
+  // Minimum motion to be considered as a shot boundary frame.
+  optional double min_motion = 1 [default = 0.2];
+  // Minimum number of shot duration (in seconds).
+  optional double min_shot_span = 2 [default = 2];
+  // A window for computing shot measure (see the definition in min_shot_measure
+  // field).
+  optional int32 window_size = 3 [default = 7];
+  // Minimum shot measure to be considered as a shot boundary frame.
+  // Must also satisfy the min_motion_with_shot_measure constraint.
+  // The shot measure is defined as the ratio of the motion of the
+  // current frame to the maximum motion of the frames in the window (defined
+  // as window_size).
+  optional double min_shot_measure = 4 [default = 10];
+  // Minimum motion to be considered as a shot boundary frame.
+  // Must also satisfy the min_shot_measure constraint.
+  optional double min_motion_with_shot_measure = 5 [default = 0.05];
+  // Only send results if the shot value is true.
+  optional bool output_only_on_change = 6 [default = true];
+  // Perform histogram equalization before computing keypoints/features.
+  optional bool equalize_histogram = 7 [default = false];
+}

mediapipe/examples/desktop/autoflip/calculators/shot_boundary_calculator_test.cc

@@ -0,0 +1,163 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "absl/strings/string_view.h"
+#include "mediapipe/examples/desktop/autoflip/calculators/shot_boundary_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/deps/file_path.h"
+#include "mediapipe/framework/formats/image_frame.h"
+#include "mediapipe/framework/formats/image_frame_opencv.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/opencv_core_inc.h"
+#include "mediapipe/framework/port/opencv_imgcodecs_inc.h"
+#include "mediapipe/framework/port/opencv_imgproc_inc.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/port/status_matchers.h"
+
+using mediapipe::Adopt;
+using mediapipe::CalculatorGraphConfig;
+using mediapipe::CalculatorRunner;
+using mediapipe::ImageFormat;
+using mediapipe::ImageFrame;
+using mediapipe::PacketTypeSet;
+using mediapipe::ParseTextProtoOrDie;
+using mediapipe::Timestamp;
+
+namespace mediapipe {
+namespace autoflip {
+namespace {
+
+const char kConfig[] = R"(
+    calculator: "ShotBoundaryCalculator"
+    input_stream: "VIDEO:camera_frames"
+    output_stream: "IS_SHOT_CHANGE:is_shot"
+    )";
+const int kTestFrameWidth = 640;
+const int kTestFrameHeight = 480;
+
+void AddFrames(const int number_of_frames, const std::set<int>& skip_frames,
+               CalculatorRunner* runner) {
+  cv::Mat image =
+      cv::imread(file::JoinPath("./",
+                                "/mediapipe/examples/desktop/"
+                                "autoflip/calculators/testdata/dino.jpg"));
+
+  for (int i = 0; i < number_of_frames; i++) {
+    auto input_frame = ::absl::make_unique<ImageFrame>(
+        ImageFormat::SRGB, kTestFrameWidth, kTestFrameHeight);
+    cv::Mat input_mat = mediapipe::formats::MatView(input_frame.get());
+    input_mat.setTo(cv::Scalar(0, 0, 0));
+    cv::Mat sub_image =
+        image(cv::Rect(i, i, kTestFrameWidth, kTestFrameHeight));
+    cv::Mat frame_area =
+        input_mat(cv::Rect(0, 0, sub_image.cols, sub_image.rows));
+    if (skip_frames.count(i) < 1) {
+      sub_image.copyTo(frame_area);
+    }
+    runner->MutableInputs()->Tag("VIDEO").packets.push_back(
+        Adopt(input_frame.release()).At(Timestamp(i * 1000000)));
+  }
+}
+
+void CheckOutput(const int number_of_frames, const std::set<int>& shot_frames,
+                 const std::vector<Packet>& output_packets) {
+  ASSERT_EQ(number_of_frames, output_packets.size());
+  for (int i = 0; i < number_of_frames; i++) {
+    if (shot_frames.count(i) < 1) {
+      EXPECT_FALSE(output_packets[i].Get<bool>());
+    } else {
+      EXPECT_TRUE(output_packets[i].Get<bool>());
+    }
+  }
+}
+
+TEST(ShotBoundaryCalculatorTest, NoShotChange) {
+  CalculatorGraphConfig::Node node =
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfig);
+  node.mutable_options()
+      ->MutableExtension(ShotBoundaryCalculatorOptions::ext)
+      ->set_output_only_on_change(false);
+  auto runner = ::absl::make_unique<CalculatorRunner>(node);
+
+  AddFrames(10, {}, runner.get());
+  MP_ASSERT_OK(runner->Run());
+  CheckOutput(10, {}, runner->Outputs().Tag("IS_SHOT_CHANGE").packets);
+}
+
+TEST(ShotBoundaryCalculatorTest, ShotChangeSingle) {
+  CalculatorGraphConfig::Node node =
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfig);
+  node.mutable_options()
+      ->MutableExtension(ShotBoundaryCalculatorOptions::ext)
+      ->set_output_only_on_change(false);
+  auto runner = ::absl::make_unique<CalculatorRunner>(node);
+
+  AddFrames(20, {10}, runner.get());
+  MP_ASSERT_OK(runner->Run());
+  CheckOutput(20, {10}, runner->Outputs().Tag("IS_SHOT_CHANGE").packets);
+}
+
+TEST(ShotBoundaryCalculatorTest, ShotChangeDouble) {
+  CalculatorGraphConfig::Node node =
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfig);
+  node.mutable_options()
+      ->MutableExtension(ShotBoundaryCalculatorOptions::ext)
+      ->set_output_only_on_change(false);
+  auto runner = ::absl::make_unique<CalculatorRunner>(node);
+
+  AddFrames(20, {14, 17}, runner.get());
+  MP_ASSERT_OK(runner->Run());
+  CheckOutput(20, {14, 17}, runner->Outputs().Tag("IS_SHOT_CHANGE").packets);
+}
+
+TEST(ShotBoundaryCalculatorTest, ShotChangeFiltered) {
+  CalculatorGraphConfig::Node node =
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfig);
+  node.mutable_options()
+      ->MutableExtension(ShotBoundaryCalculatorOptions::ext)
+      ->set_min_shot_span(5);
+  node.mutable_options()
+      ->MutableExtension(ShotBoundaryCalculatorOptions::ext)
+      ->set_output_only_on_change(false);
+
+  auto runner = ::absl::make_unique<CalculatorRunner>(node);
+
+  AddFrames(24, {16, 19}, runner.get());
+  MP_ASSERT_OK(runner->Run());
+  CheckOutput(24, {16}, runner->Outputs().Tag("IS_SHOT_CHANGE").packets);
+}
+
+TEST(ShotBoundaryCalculatorTest, ShotChangeSingleOnOnChange) {
+  CalculatorGraphConfig::Node node =
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfig);
+  node.mutable_options()
+      ->MutableExtension(ShotBoundaryCalculatorOptions::ext)
+      ->set_output_only_on_change(true);
+  auto runner = ::absl::make_unique<CalculatorRunner>(node);
+
+  AddFrames(20, {15}, runner.get());
+  MP_ASSERT_OK(runner->Run());
+  auto output_packets = runner->Outputs().Tag("IS_SHOT_CHANGE").packets;
+  ASSERT_EQ(output_packets.size(), 1);
+  ASSERT_EQ(output_packets[0].Get<bool>(), true);
+  ASSERT_EQ(output_packets[0].Timestamp().Value(), 15000000);
+}
+
+}  // namespace
+}  // namespace autoflip
+}  // namespace mediapipe

mediapipe/examples/desktop/autoflip/calculators/signal_fusing_calculator.cc

@@ -0,0 +1,231 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <map>
+#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "mediapipe/examples/desktop/autoflip/autoflip_messages.pb.h"
+#include "mediapipe/examples/desktop/autoflip/calculators/signal_fusing_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+
+using mediapipe::Packet;
+using mediapipe::PacketTypeSet;
+using mediapipe::autoflip::DetectionSet;
+using mediapipe::autoflip::SalientRegion;
+using mediapipe::autoflip::SignalType;
+
+namespace mediapipe {
+namespace autoflip {
+
+struct InputSignal {
+  SalientRegion signal;
+  int source;
+};
+
+struct Frame {
+  std::vector<InputSignal> input_detections;
+  mediapipe::Timestamp time;
+};
+
+// This calculator takes one scene change signal and an arbitrary number of
+// detection signals and outputs a single list of detections.  The scores for
+// the detections can be re-normalized using the options proto.  Additionally,
+// if a detection has a consistent tracking id during a scene the score for that
+// detection is averaged over the whole scene.
+//
+// Example:
+//  node {
+//    calculator: "SignalFusingCalculator"
+//    input_stream: "scene_change"
+//    input_stream: "detection_faces"
+//    input_stream: "detection_custom_text"
+//    output_stream: "salient_region"
+//    options:{
+//    [mediapipe.autoflip.SignalFusingCalculatorOptions.ext]:{
+//      signal_settings{
+//        type: {standard: FACE}
+//        min_score: 0.5
+//        max_score: 0.6
+//      }
+//      signal_settings{
+//        type: {custom: "custom_text"}
+//        min_score: 0.9
+//        max_score: 1.0
+//      }
+//    }
+//    }
+//  }
+class SignalFusingCalculator : public mediapipe::CalculatorBase {
+ public:
+  SignalFusingCalculator() {}
+  SignalFusingCalculator(const SignalFusingCalculator&) = delete;
+  SignalFusingCalculator& operator=(const SignalFusingCalculator&) = delete;
+
+  static ::mediapipe::Status GetContract(mediapipe::CalculatorContract* cc);
+  mediapipe::Status Open(mediapipe::CalculatorContext* cc) override;
+  mediapipe::Status Process(mediapipe::CalculatorContext* cc) override;
+  mediapipe::Status Close(mediapipe::CalculatorContext* cc) override;
+
+ private:
+  mediapipe::Status ProcessScene(mediapipe::CalculatorContext* cc);
+  SignalFusingCalculatorOptions options_;
+  std::map<std::string, SignalSettings> settings_by_type_;
+  std::vector<Frame> scene_frames_;
+};
+REGISTER_CALCULATOR(SignalFusingCalculator);
+
+namespace {
+std::string CreateSettingsKey(const SignalType& signal_type) {
+  if (signal_type.has_standard()) {
+    return "standard_" + std::to_string(signal_type.standard());
+  } else {
+    return "custom_" + signal_type.custom();
+  }
+}
+std::string CreateKey(const InputSignal& detection) {
+  std::string id_source = std::to_string(detection.source);
+  std::string id_signal = std::to_string(detection.signal.tracking_id());
+  std::string id = id_source + ":" + id_signal;
+  return id;
+}
+
+}  // namespace
+
+mediapipe::Status SignalFusingCalculator::Open(
+    mediapipe::CalculatorContext* cc) {
+  options_ = cc->Options<SignalFusingCalculatorOptions>();
+  for (const auto& setting : options_.signal_settings()) {
+    settings_by_type_[CreateSettingsKey(setting.type())] = setting;
+  }
+  return ::mediapipe::OkStatus();
+}
+
+mediapipe::Status SignalFusingCalculator::Close(
+    mediapipe::CalculatorContext* cc) {
+  if (!scene_frames_.empty()) {
+    MP_RETURN_IF_ERROR(ProcessScene(cc));
+    scene_frames_.clear();
+  }
+  return ::mediapipe::OkStatus();
+}
+
+mediapipe::Status SignalFusingCalculator::ProcessScene(
+    mediapipe::CalculatorContext* cc) {
+  std::map<std::string, int> detection_count;
+  std::map<std::string, float> multiframe_score;
+  // Create a unified score for all items with temporal ids.
+  for (const Frame& frame : scene_frames_) {
+    for (const auto& detection : frame.input_detections) {
+      if (detection.signal.has_tracking_id()) {
+        // Create key for each detector type
+        if (detection_count.find(CreateKey(detection)) ==
+            detection_count.end()) {
+          multiframe_score[CreateKey(detection)] = 0.0;
+          detection_count[CreateKey(detection)] = 0;
+        }
+        multiframe_score[CreateKey(detection)] += detection.signal.score();
+        detection_count[CreateKey(detection)]++;
+      }
+    }
+  }
+
+  // Average scores.
+  for (auto iterator = multiframe_score.begin();
+       iterator != multiframe_score.end(); iterator++) {
+    multiframe_score[iterator->first] =
+        iterator->second / detection_count[iterator->first];
+  }
+
+  // Process detections.
+  for (const Frame& frame : scene_frames_) {
+    std::unique_ptr<DetectionSet> processed_detections(new DetectionSet());
+    for (auto detection : frame.input_detections) {
+      float score = detection.signal.score();
+      if (detection.signal.has_tracking_id()) {
+        std::string id_source = std::to_string(detection.source);
+        std::string id_signal = std::to_string(detection.signal.tracking_id());
+        std::string id = id_source + ":" + id_signal;
+        score = multiframe_score[id];
+      }
+      // Normalize within range.
+      float min_value = 0.0;
+      float max_value = 1.0;
+
+      auto settings_it = settings_by_type_.find(
+          CreateSettingsKey(detection.signal.signal_type()));
+      if (settings_it != settings_by_type_.end()) {
+        min_value = settings_it->second.min_score();
+        max_value = settings_it->second.max_score();
+        detection.signal.set_is_required(settings_it->second.is_required());
+      }
+
+      float final_score = score * (max_value - min_value) + min_value;
+      detection.signal.set_score(final_score);
+      *processed_detections->add_detections() = detection.signal;
+    }
+    cc->Outputs().Index(0).Add(processed_detections.release(), frame.time);
+  }
+
+  return ::mediapipe::OkStatus();
+}
+
+mediapipe::Status SignalFusingCalculator::Process(
+    mediapipe::CalculatorContext* cc) {
+  bool is_boundary = false;
+  if (!cc->Inputs().Index(0).Value().IsEmpty()) {
+    is_boundary = cc->Inputs().Index(0).Get<bool>();
+  }
+
+  if (is_boundary || scene_frames_.size() > options_.max_scene_size()) {
+    MP_RETURN_IF_ERROR(ProcessScene(cc));
+    scene_frames_.clear();
+  }
+
+  Frame frame;
+  for (int i = 1; i < cc->Inputs().NumEntries(); ++i) {
+    const Packet& packet = cc->Inputs().Index(i).Value();
+    if (packet.IsEmpty()) {
+      continue;
+    }
+    const auto& detection_set = packet.Get<autoflip::DetectionSet>();
+    for (const auto& detection : detection_set.detections()) {
+      InputSignal input;
+      input.signal = detection;
+      input.source = i;
+      frame.input_detections.push_back(input);
+    }
+  }
+  frame.time = cc->InputTimestamp();
+  scene_frames_.push_back(frame);
+
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status SignalFusingCalculator::GetContract(
+    mediapipe::CalculatorContract* cc) {
+  cc->Inputs().Index(0).Set<bool>();
+  for (int i = 1; i < cc->Inputs().NumEntries(); ++i) {
+    cc->Inputs().Index(i).Set<autoflip::DetectionSet>();
+  }
+  cc->Outputs().Index(0).Set<autoflip::DetectionSet>();
+  return ::mediapipe::OkStatus();
+}
+
+}  // namespace autoflip
+}  // namespace mediapipe

mediapipe/examples/desktop/autoflip/calculators/signal_fusing_calculator.proto

@@ -0,0 +1,54 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe.autoflip;
+
+import "mediapipe/examples/desktop/autoflip/autoflip_messages.proto";
+import "mediapipe/framework/calculator.proto";
+
+// Next tag: 3
+message SignalFusingCalculatorOptions {
+  extend mediapipe.CalculatorOptions {
+    optional SignalFusingCalculatorOptions ext = 280092372;
+  }
+  // Setting related to each type of signal this calculator could process.
+  repeated SignalSettings signal_settings = 1;
+
+  // Force a flush of the frame buffer after this number of frames.
+  optional int32 max_scene_size = 2 [default = 600];
+}
+
+// Next tag: 5
+message SignalSettings {
+  // The type of signal these settings pertain to.
+  optional SignalType type = 1;
+
+  // Force a normalized incoming score to be re-normalized to within this range.
+  // (set values to min:0 and max:1 for no change in the incoming score)
+  // Values must be between 0-1, min must be less than max.
+  //
+  // Example of score adjustment:
+  // Incoming OCR score: .7
+  // Min OCR Score: .9
+  // Max OCR Score: 1.0
+  // --Result: .97
+  optional float min_score = 2 [default = 0];
+  optional float max_score = 3 [default = 1.0];
+
+  // Is this signal required within the output cropped video?  If it is it will
+  // be included or the video will be marked as failed to convert.
+  optional bool is_required = 4 [default = false];
+}

mediapipe/examples/desktop/autoflip/calculators/signal_fusing_calculator_test.cc

@@ -0,0 +1,438 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "absl/strings/string_view.h"
+#include "mediapipe/examples/desktop/autoflip/autoflip_messages.pb.h"
+#include "mediapipe/examples/desktop/autoflip/calculators/signal_fusing_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/formats/image_frame.h"
+#include "mediapipe/framework/formats/image_frame_opencv.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/port/status_matchers.h"
+
+using mediapipe::autoflip::DetectionSet;
+
+namespace mediapipe {
+namespace autoflip {
+namespace {
+
+const char kConfigA[] = R"(
+    calculator: "SignalFusingCalculator"
+    input_stream: "scene_change"
+    input_stream: "detection_set_a"
+    input_stream: "detection_set_b"
+    output_stream: "salient_region"
+    options:{
+    [mediapipe.autoflip.SignalFusingCalculatorOptions.ext]:{
+      signal_settings{
+        type: {standard: FACE_FULL}
+        min_score: 0.5
+        max_score: 0.6
+      }
+      signal_settings{
+        type: {standard: TEXT}
+        min_score: 0.9
+        max_score: 1.0
+      }
+    }
+    })";
+
+const char kConfigB[] = R"(
+    calculator: "SignalFusingCalculator"
+    input_stream: "scene_change"
+    input_stream: "detection_set_a"
+    input_stream: "detection_set_b"
+    input_stream: "detection_set_c"
+    output_stream: "salient_region"
+    options:{
+    [mediapipe.autoflip.SignalFusingCalculatorOptions.ext]:{
+      signal_settings{
+        type: {standard: FACE_FULL}
+        min_score: 0.5
+        max_score: 0.6
+      }
+      signal_settings{
+        type: {custom: "text"}
+        min_score: 0.9
+        max_score: 1.0
+      }
+      signal_settings{
+        type: {standard: LOGO}
+        min_score: 0.1
+        max_score: 0.3
+      }
+    }
+    })";
+
+TEST(SignalFusingCalculatorTest, TwoInputNoTracking) {
+  auto runner = absl::make_unique<CalculatorRunner>(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfigA));
+
+  auto input_border = absl::make_unique<bool>(false);
+  runner->MutableInputs()->Index(0).packets.push_back(
+      Adopt(input_border.release()).At(Timestamp(0)));
+
+  auto input_face =
+      absl::make_unique<DetectionSet>(ParseTextProtoOrDie<DetectionSet>(
+          R"(
+            detections {
+              score: 0.5
+              signal_type: { standard: FACE_FULL }
+            }
+            detections {
+              score: 0.3
+              signal_type: { standard: FACE_FULL }
+            }
+          )"));
+
+  runner->MutableInputs()->Index(1).packets.push_back(
+      Adopt(input_face.release()).At(Timestamp(0)));
+
+  auto input_ocr =
+      absl::make_unique<DetectionSet>(ParseTextProtoOrDie<DetectionSet>(
+          R"(
+            detections {
+              score: 0.3
+              signal_type: { standard: TEXT }
+            }
+            detections {
+              score: 0.9
+              signal_type: { standard: TEXT }
+            }
+          )"));
+
+  runner->MutableInputs()->Index(2).packets.push_back(
+      Adopt(input_ocr.release()).At(Timestamp(0)));
+
+  MP_ASSERT_OK(runner->Run());
+
+  const std::vector<Packet>& output_packets =
+      runner->Outputs().Index(0).packets;
+  const auto& detection_set = output_packets[0].Get<DetectionSet>();
+
+  ASSERT_EQ(detection_set.detections().size(), 4);
+  EXPECT_FLOAT_EQ(detection_set.detections(0).score(), .55);
+  EXPECT_FLOAT_EQ(detection_set.detections(1).score(), .53);
+  EXPECT_FLOAT_EQ(detection_set.detections(2).score(), .93);
+  EXPECT_FLOAT_EQ(detection_set.detections(3).score(), .99);
+}
+
+TEST(SignalFusingCalculatorTest, ThreeInputTracking) {
+  auto runner = absl::make_unique<CalculatorRunner>(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfigB));
+
+  auto input_border_0 = absl::make_unique<bool>(false);
+  runner->MutableInputs()->Index(0).packets.push_back(
+      Adopt(input_border_0.release()).At(Timestamp(0)));
+
+  // Time zero.
+  auto input_face_0 =
+      absl::make_unique<DetectionSet>(ParseTextProtoOrDie<DetectionSet>(
+          R"(
+            detections {
+              score: 0.2
+              signal_type: { standard: FACE_FULL }
+              tracking_id: 0
+            }
+            detections {
+              score: 0.0
+              signal_type: { standard: FACE_FULL }
+              tracking_id: 1
+            }
+            detections {
+              score: 0.1
+              signal_type: { standard: FACE_FULL }
+            }
+          )"));
+
+  runner->MutableInputs()->Index(1).packets.push_back(
+      Adopt(input_face_0.release()).At(Timestamp(0)));
+
+  auto input_ocr_0 =
+      absl::make_unique<DetectionSet>(ParseTextProtoOrDie<DetectionSet>(
+          R"(
+            detections {
+              score: 0.2
+              signal_type: { custom: "text" }
+            }
+          )"));
+
+  runner->MutableInputs()->Index(2).packets.push_back(
+      Adopt(input_ocr_0.release()).At(Timestamp(0)));
+
+  auto input_agn_0 =
+      absl::make_unique<DetectionSet>(ParseTextProtoOrDie<DetectionSet>(
+          R"(
+            detections {
+              score: 0.3
+              signal_type: { standard: LOGO }
+              tracking_id: 0
+            }
+          )"));
+
+  runner->MutableInputs()->Index(3).packets.push_back(
+      Adopt(input_agn_0.release()).At(Timestamp(0)));
+
+  // Time one
+  auto input_border_1 = absl::make_unique<bool>(false);
+  runner->MutableInputs()->Index(0).packets.push_back(
+      Adopt(input_border_1.release()).At(Timestamp(1)));
+
+  auto input_face_1 =
+      absl::make_unique<DetectionSet>(ParseTextProtoOrDie<DetectionSet>(
+          R"(
+            detections {
+              score: 0.7
+              signal_type: { standard: FACE_FULL }
+              tracking_id: 0
+            }
+            detections {
+              score: 0.9
+              signal_type: { standard: FACE_FULL }
+              tracking_id: 1
+            }
+            detections {
+              score: 0.2
+              signal_type: { standard: FACE_FULL }
+            }
+          )"));
+
+  runner->MutableInputs()->Index(1).packets.push_back(
+      Adopt(input_face_1.release()).At(Timestamp(1)));
+
+  auto input_ocr_1 =
+      absl::make_unique<DetectionSet>(ParseTextProtoOrDie<DetectionSet>(
+          R"(
+            detections {
+              score: 0.3
+              signal_type: { custom: "text" }
+            }
+          )"));
+
+  runner->MutableInputs()->Index(2).packets.push_back(
+      Adopt(input_ocr_1.release()).At(Timestamp(1)));
+
+  auto input_agn_1 =
+      absl::make_unique<DetectionSet>(ParseTextProtoOrDie<DetectionSet>(
+          R"(
+            detections {
+              score: 0.3
+              signal_type: { standard: LOGO }
+              tracking_id: 0
+            }
+          )"));
+
+  runner->MutableInputs()->Index(3).packets.push_back(
+      Adopt(input_agn_1.release()).At(Timestamp(1)));
+
+  // Time two
+  auto input_border_2 = absl::make_unique<bool>(false);
+  runner->MutableInputs()->Index(0).packets.push_back(
+      Adopt(input_border_2.release()).At(Timestamp(2)));
+
+  auto input_face_2 =
+      absl::make_unique<DetectionSet>(ParseTextProtoOrDie<DetectionSet>(
+          R"(
+            detections {
+              score: 0.8
+              signal_type: { standard: FACE_FULL }
+              tracking_id: 0
+            }
+            detections {
+              score: 0.9
+              signal_type: { standard: FACE_FULL }
+              tracking_id: 1
+            }
+            detections {
+              score: 0.3
+              signal_type: { standard: FACE_FULL }
+            }
+          )"));
+
+  runner->MutableInputs()->Index(1).packets.push_back(
+      Adopt(input_face_2.release()).At(Timestamp(2)));
+
+  auto input_ocr_2 =
+      absl::make_unique<DetectionSet>(ParseTextProtoOrDie<DetectionSet>(
+          R"(
+            detections {
+              score: 0.3
+              signal_type: { custom: "text" }
+            }
+          )"));
+
+  runner->MutableInputs()->Index(2).packets.push_back(
+      Adopt(input_ocr_2.release()).At(Timestamp(2)));
+
+  auto input_agn_2 =
+      absl::make_unique<DetectionSet>(ParseTextProtoOrDie<DetectionSet>(
+          R"(
+            detections {
+              score: 0.9
+              signal_type: { standard: LOGO }
+              tracking_id: 0
+            }
+          )"));
+
+  runner->MutableInputs()->Index(3).packets.push_back(
+      Adopt(input_agn_2.release()).At(Timestamp(2)));
+
+  // Time three (new scene)
+  auto input_border_3 = absl::make_unique<bool>(true);
+  runner->MutableInputs()->Index(0).packets.push_back(
+      Adopt(input_border_3.release()).At(Timestamp(3)));
+
+  auto input_face_3 =
+      absl::make_unique<DetectionSet>(ParseTextProtoOrDie<DetectionSet>(
+          R"(
+            detections {
+              score: 0.2
+              signal_type: { standard: FACE_FULL }
+              tracking_id: 0
+            }
+            detections {
+              score: 0.3
+              signal_type: { standard: FACE_FULL }
+              tracking_id: 1
+            }
+            detections {
+              score: 0.4
+              signal_type: { standard: FACE_FULL }
+            }
+          )"));
+
+  runner->MutableInputs()->Index(1).packets.push_back(
+      Adopt(input_face_3.release()).At(Timestamp(3)));
+
+  auto input_ocr_3 =
+      absl::make_unique<DetectionSet>(ParseTextProtoOrDie<DetectionSet>(
+          R"(
+            detections {
+              score: 0.5
+              signal_type: { custom: "text" }
+            }
+          )"));
+
+  runner->MutableInputs()->Index(2).packets.push_back(
+      Adopt(input_ocr_3.release()).At(Timestamp(3)));
+
+  auto input_agn_3 =
+      absl::make_unique<DetectionSet>(ParseTextProtoOrDie<DetectionSet>(
+          R"(
+            detections {
+              score: 0.6
+              signal_type: { standard: LOGO }
+              tracking_id: 0
+            }
+          )"));
+
+  runner->MutableInputs()->Index(3).packets.push_back(
+      Adopt(input_agn_3.release()).At(Timestamp(3)));
+
+  MP_ASSERT_OK(runner->Run());
+
+  // Check time 0
+  std::vector<Packet> output_packets = runner->Outputs().Index(0).packets;
+  DetectionSet detection_set = output_packets[0].Get<DetectionSet>();
+
+  float face_id_0 = (.2 + .7 + .8) / 3;
+  face_id_0 = face_id_0 * .1 + .5;
+  float face_id_1 = (0.0 + .9 + .9) / 3;
+  face_id_1 = face_id_1 * .1 + .5;
+  float face_3 = 0.1;
+  face_3 = face_3 * .1 + .5;
+  float ocr_1 = 0.2;
+  ocr_1 = ocr_1 * .1 + .9;
+  float agn_1 = (.3 + .3 + .9) / 3;
+  agn_1 = agn_1 * .2 + .1;
+
+  ASSERT_EQ(detection_set.detections().size(), 5);
+  EXPECT_FLOAT_EQ(detection_set.detections(0).score(), face_id_0);
+  EXPECT_FLOAT_EQ(detection_set.detections(1).score(), face_id_1);
+  EXPECT_FLOAT_EQ(detection_set.detections(2).score(), face_3);
+  EXPECT_FLOAT_EQ(detection_set.detections(3).score(), ocr_1);
+  EXPECT_FLOAT_EQ(detection_set.detections(4).score(), agn_1);
+
+  // Check time 1
+  detection_set = output_packets[1].Get<DetectionSet>();
+
+  face_id_0 = (.2 + .7 + .8) / 3;
+  face_id_0 = face_id_0 * .1 + .5;
+  face_id_1 = (0.0 + .9 + .9) / 3;
+  face_id_1 = face_id_1 * .1 + .5;
+  face_3 = 0.2;
+  face_3 = face_3 * .1 + .5;
+  ocr_1 = 0.3;
+  ocr_1 = ocr_1 * .1 + .9;
+  agn_1 = (.3 + .3 + .9) / 3;
+  agn_1 = agn_1 * .2 + .1;
+
+  ASSERT_EQ(detection_set.detections().size(), 5);
+  EXPECT_FLOAT_EQ(detection_set.detections(0).score(), face_id_0);
+  EXPECT_FLOAT_EQ(detection_set.detections(1).score(), face_id_1);
+  EXPECT_FLOAT_EQ(detection_set.detections(2).score(), face_3);
+  EXPECT_FLOAT_EQ(detection_set.detections(3).score(), ocr_1);
+  EXPECT_FLOAT_EQ(detection_set.detections(4).score(), agn_1);
+
+  // Check time 2
+  detection_set = output_packets[2].Get<DetectionSet>();
+
+  face_id_0 = (.2 + .7 + .8) / 3;
+  face_id_0 = face_id_0 * .1 + .5;
+  face_id_1 = (0.0 + .9 + .9) / 3;
+  face_id_1 = face_id_1 * .1 + .5;
+  face_3 = 0.3;
+  face_3 = face_3 * .1 + .5;
+  ocr_1 = 0.3;
+  ocr_1 = ocr_1 * .1 + .9;
+  agn_1 = (.3 + .3 + .9) / 3;
+  agn_1 = agn_1 * .2 + .1;
+
+  ASSERT_EQ(detection_set.detections().size(), 5);
+  EXPECT_FLOAT_EQ(detection_set.detections(0).score(), face_id_0);
+  EXPECT_FLOAT_EQ(detection_set.detections(1).score(), face_id_1);
+  EXPECT_FLOAT_EQ(detection_set.detections(2).score(), face_3);
+  EXPECT_FLOAT_EQ(detection_set.detections(3).score(), ocr_1);
+  EXPECT_FLOAT_EQ(detection_set.detections(4).score(), agn_1);
+
+  // Check time 3 (new scene)
+  detection_set = output_packets[3].Get<DetectionSet>();
+
+  face_id_0 = 0.2;
+  face_id_0 = face_id_0 * .1 + .5;
+  face_id_1 = 0.3;
+  face_id_1 = face_id_1 * .1 + .5;
+  face_3 = 0.4;
+  face_3 = face_3 * .1 + .5;
+  ocr_1 = 0.5;
+  ocr_1 = ocr_1 * .1 + .9;
+  agn_1 = .6;
+  agn_1 = agn_1 * .2 + .1;
+
+  ASSERT_EQ(detection_set.detections().size(), 5);
+  EXPECT_FLOAT_EQ(detection_set.detections(0).score(), face_id_0);
+  EXPECT_FLOAT_EQ(detection_set.detections(1).score(), face_id_1);
+  EXPECT_FLOAT_EQ(detection_set.detections(2).score(), face_3);
+  EXPECT_FLOAT_EQ(detection_set.detections(3).score(), ocr_1);
+  EXPECT_FLOAT_EQ(detection_set.detections(4).score(), agn_1);
+}
+
+}  // namespace
+}  // namespace autoflip
+}  // namespace mediapipe

mediapipe/examples/desktop/autoflip/calculators/testdata/BUILD

@@ -0,0 +1,23 @@
+# Copyright 2019 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+licenses(["notice"])  # Apache 2.0
+
+filegroup(
+    name = "test_images",
+    srcs = [
+        "dino.jpg",
+    ],
+    visibility = ["//visibility:public"],
+)

mediapipe/examples/desktop/autoflip/calculators/video_filtering_calculator.cc

@@ -0,0 +1,121 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <string>
+
+#include "absl/strings/string_view.h"
+#include "absl/strings/substitute.h"
+#include "mediapipe/examples/desktop/autoflip/calculators/video_filtering_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/image_frame.h"
+#include "mediapipe/framework/formats/image_frame_opencv.h"
+#include "mediapipe/framework/port/status_builder.h"
+
+namespace mediapipe {
+namespace autoflip {
+namespace {
+constexpr char kInputFrameTag[] = "INPUT_FRAMES";
+constexpr char kOutputFrameTag[] = "OUTPUT_FRAMES";
+}  // namespace
+
+// This calculator filters out frames based on criteria specified in the
+// options. One use case is to filter based on the aspect ratio. Future work
+// can implement more filter types.
+//
+// Input: Video frames.
+// Output: Video frames that pass all filters.
+//
+// Example config:
+// node {
+//   calculator: "VideoFilteringCalculator"
+//   input_stream: "INPUT_FRAMES:frames"
+//   output_stream: "OUTPUT_FRAMES:output_frames"
+//   options: {
+//     [mediapipe.autoflip.VideoFilteringCalculatorOptions.ext]: {
+//       fail_if_any: true
+//       aspect_ratio_filter {
+//         target_width: 400
+//         target_height: 600
+//         filter_type: UPPER_ASPECT_RATIO_THRESHOLD
+//       }
+//     }
+//   }
+// }
+class VideoFilteringCalculator : public CalculatorBase {
+ public:
+  VideoFilteringCalculator() = default;
+  ~VideoFilteringCalculator() override = default;
+
+  static ::mediapipe::Status GetContract(CalculatorContract* cc);
+
+  ::mediapipe::Status Process(CalculatorContext* cc) override;
+};
+REGISTER_CALCULATOR(VideoFilteringCalculator);
+
+::mediapipe::Status VideoFilteringCalculator::GetContract(
+    CalculatorContract* cc) {
+  cc->Inputs().Tag(kInputFrameTag).Set<ImageFrame>();
+  cc->Outputs().Tag(kOutputFrameTag).Set<ImageFrame>();
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status VideoFilteringCalculator::Process(CalculatorContext* cc) {
+  const auto& options = cc->Options<VideoFilteringCalculatorOptions>();
+
+  const Packet& input_packet = cc->Inputs().Tag(kInputFrameTag).Value();
+  const ImageFrame& frame = input_packet.Get<ImageFrame>();
+
+  RET_CHECK(options.has_aspect_ratio_filter());
+  const auto filter_type = options.aspect_ratio_filter().filter_type();
+  RET_CHECK_NE(
+      filter_type,
+      VideoFilteringCalculatorOptions::AspectRatioFilter::UNKNOWN_FILTER_TYPE);
+  if (filter_type ==
+      VideoFilteringCalculatorOptions::AspectRatioFilter::NO_FILTERING) {
+    cc->Outputs().Tag(kOutputFrameTag).AddPacket(input_packet);
+    return ::mediapipe::OkStatus();
+  }
+  const int target_width = options.aspect_ratio_filter().target_width();
+  const int target_height = options.aspect_ratio_filter().target_height();
+  RET_CHECK_GT(target_width, 0);
+  RET_CHECK_GT(target_height, 0);
+
+  bool should_pass = false;
+  cv::Mat frame_mat = ::mediapipe::formats::MatView(&frame);
+  const double ratio = static_cast<double>(frame_mat.cols) / frame_mat.rows;
+  const double target_ratio = static_cast<double>(target_width) / target_height;
+  if (filter_type == VideoFilteringCalculatorOptions::AspectRatioFilter::
+                         UPPER_ASPECT_RATIO_THRESHOLD &&
+      ratio <= target_ratio) {
+    should_pass = true;
+  } else if (filter_type == VideoFilteringCalculatorOptions::AspectRatioFilter::
+                                LOWER_ASPECT_RATIO_THRESHOLD &&
+             ratio >= target_ratio) {
+    should_pass = true;
+  }
+  if (should_pass) {
+    cc->Outputs().Tag(kOutputFrameTag).AddPacket(input_packet);
+    return ::mediapipe::OkStatus();
+  }
+  if (options.fail_if_any()) {
+    return ::mediapipe::UnknownErrorBuilder(MEDIAPIPE_LOC) << absl::Substitute(
+               "Failing due to aspect ratio. Target aspect ratio: $0. Frame "
+               "width: $1, height: $2.",
+               target_ratio, frame.Width(), frame.Height());
+  }
+
+  return ::mediapipe::OkStatus();
+}
+}  // namespace autoflip
+}  // namespace mediapipe

mediapipe/examples/desktop/autoflip/calculators/video_filtering_calculator.proto

@@ -0,0 +1,56 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe.autoflip;
+
+import "mediapipe/framework/calculator.proto";
+
+message VideoFilteringCalculatorOptions {
+  extend mediapipe.CalculatorOptions {
+    optional VideoFilteringCalculatorOptions ext = 278504113;
+  }
+
+  // If true, when an input frame needs be filtered out according to the filter
+  // type and conditions, the calculator would return a FAIL status. Otherwise,
+  // the calculator would simply skip the filtered frames and would not pass it
+  // down to downstream nodes.
+  optional bool fail_if_any = 1 [default = false];
+
+  message AspectRatioFilter {
+    // Target width and height, which define the aspect ratio
+    // (i.e. target_width / target_height) to compare input frames with. The
+    // actual values of these fields do not matter, only the ratio between them
+    // does. These values must be set to positive.
+    optional int32 target_width = 1 [default = -1];
+    optional int32 target_height = 2 [default = -1];
+    enum FilterType {
+      UNKNOWN_FILTER_TYPE = 0;
+      // Use this type when the target width and height defines an upper bound
+      // (inclusive) of the aspect ratio.
+      UPPER_ASPECT_RATIO_THRESHOLD = 1;
+      // Use this type when the target width and height defines a lower bound
+      // (inclusive) of the aspect ratio.
+      LOWER_ASPECT_RATIO_THRESHOLD = 2;
+      // Use this type to configure the calculator as a no-op pass-through node.
+      NO_FILTERING = 3;
+    }
+    optional FilterType filter_type = 3;
+  }
+
+  oneof filter {
+    AspectRatioFilter aspect_ratio_filter = 2;
+  }
+}

mediapipe/examples/desktop/autoflip/calculators/video_filtering_calculator_test.cc

@@ -0,0 +1,177 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "absl/strings/string_view.h"
+#include "absl/strings/substitute.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/formats/image_frame.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/status_builder.h"
+#include "mediapipe/framework/port/status_matchers.h"
+
+namespace mediapipe {
+namespace autoflip {
+namespace {
+
+// Default configuration of the calculator.
+CalculatorGraphConfig::Node GetCalculatorNode(
+    const std::string& fail_if_any, const std::string& extra_options = "") {
+  return ParseTextProtoOrDie<CalculatorGraphConfig::Node>(
+      absl::Substitute(R"(
+        calculator: "VideoFilteringCalculator"
+        input_stream: "INPUT_FRAMES:frames"
+        output_stream: "OUTPUT_FRAMES:output_frames"
+        options: {
+          [mediapipe.autoflip.VideoFilteringCalculatorOptions.ext]: {
+            fail_if_any: $0
+            $1
+          }
+        }
+    )",
+                       fail_if_any, extra_options));
+}
+
+TEST(VideoFilterCalculatorTest, UpperBoundNoPass) {
+  CalculatorGraphConfig::Node config = GetCalculatorNode("false", R"(
+    aspect_ratio_filter {
+      target_width: 2
+      target_height: 1
+      filter_type: UPPER_ASPECT_RATIO_THRESHOLD
+    }
+  )");
+
+  auto runner = ::absl::make_unique<CalculatorRunner>(config);
+  const int kFixedWidth = 1000;
+  const double kAspectRatio = 5.0 / 1.0;
+  auto input_frame = ::absl::make_unique<ImageFrame>(
+      ImageFormat::SRGB, kFixedWidth,
+      static_cast<int>(kFixedWidth / kAspectRatio), 16);
+  runner->MutableInputs()
+      ->Tag("INPUT_FRAMES")
+      .packets.push_back(Adopt(input_frame.release()).At(Timestamp(1000)));
+  MP_ASSERT_OK(runner->Run());
+  const auto& output_packet = runner->Outputs().Tag("OUTPUT_FRAMES").packets;
+  EXPECT_TRUE(output_packet.empty());
+}
+
+TEST(VerticalFrameRemovalCalculatorTest, UpperBoundPass) {
+  CalculatorGraphConfig::Node config = GetCalculatorNode("false", R"(
+    aspect_ratio_filter {
+      target_width: 2
+      target_height: 1
+      filter_type: UPPER_ASPECT_RATIO_THRESHOLD
+    }
+  )");
+
+  auto runner = ::absl::make_unique<CalculatorRunner>(config);
+  const int kWidth = 1000;
+  const double kAspectRatio = 1.0 / 5.0;
+  const double kHeight = static_cast<int>(kWidth / kAspectRatio);
+  auto input_frame =
+      ::absl::make_unique<ImageFrame>(ImageFormat::SRGB, kWidth, kHeight, 16);
+  runner->MutableInputs()
+      ->Tag("INPUT_FRAMES")
+      .packets.push_back(Adopt(input_frame.release()).At(Timestamp(1000)));
+  MP_ASSERT_OK(runner->Run());
+  const auto& output_packet = runner->Outputs().Tag("OUTPUT_FRAMES").packets;
+  EXPECT_EQ(1, output_packet.size());
+  auto& output_frame = output_packet[0].Get<ImageFrame>();
+  EXPECT_EQ(kWidth, output_frame.Width());
+  EXPECT_EQ(kHeight, output_frame.Height());
+}
+
+TEST(VideoFilterCalculatorTest, LowerBoundNoPass) {
+  CalculatorGraphConfig::Node config = GetCalculatorNode("false", R"(
+    aspect_ratio_filter {
+      target_width: 2
+      target_height: 1
+      filter_type: LOWER_ASPECT_RATIO_THRESHOLD
+    }
+  )");
+
+  auto runner = ::absl::make_unique<CalculatorRunner>(config);
+  const int kFixedWidth = 1000;
+  const double kAspectRatio = 1.0 / 1.0;
+  auto input_frame = ::absl::make_unique<ImageFrame>(
+      ImageFormat::SRGB, kFixedWidth,
+      static_cast<int>(kFixedWidth / kAspectRatio), 16);
+  runner->MutableInputs()
+      ->Tag("INPUT_FRAMES")
+      .packets.push_back(Adopt(input_frame.release()).At(Timestamp(1000)));
+  MP_ASSERT_OK(runner->Run());
+  const auto& output_packet = runner->Outputs().Tag("OUTPUT_FRAMES").packets;
+  EXPECT_TRUE(output_packet.empty());
+}
+
+TEST(VerticalFrameRemovalCalculatorTest, LowerBoundPass) {
+  CalculatorGraphConfig::Node config = GetCalculatorNode("false", R"(
+    aspect_ratio_filter {
+      target_width: 2
+      target_height: 1
+      filter_type: LOWER_ASPECT_RATIO_THRESHOLD
+    }
+  )");
+
+  auto runner = ::absl::make_unique<CalculatorRunner>(config);
+  const int kWidth = 1000;
+  const double kAspectRatio = 5.0 / 1.0;
+  const double kHeight = static_cast<int>(kWidth / kAspectRatio);
+  auto input_frame =
+      ::absl::make_unique<ImageFrame>(ImageFormat::SRGB, kWidth, kHeight, 16);
+  runner->MutableInputs()
+      ->Tag("INPUT_FRAMES")
+      .packets.push_back(Adopt(input_frame.release()).At(Timestamp(1000)));
+  MP_ASSERT_OK(runner->Run());
+  const auto& output_packet = runner->Outputs().Tag("OUTPUT_FRAMES").packets;
+  EXPECT_EQ(1, output_packet.size());
+  auto& output_frame = output_packet[0].Get<ImageFrame>();
+  EXPECT_EQ(kWidth, output_frame.Width());
+  EXPECT_EQ(kHeight, output_frame.Height());
+}
+
+// Test that an error should be generated when fail_if_any is true.
+TEST(VerticalFrameRemovalCalculatorTest, OutputError) {
+  CalculatorGraphConfig::Node config = GetCalculatorNode("true", R"(
+    aspect_ratio_filter {
+      target_width: 2
+      target_height: 1
+      filter_type: LOWER_ASPECT_RATIO_THRESHOLD
+    }
+  )");
+
+  auto runner = ::absl::make_unique<CalculatorRunner>(config);
+  const int kFixedWidth = 1000;
+  const double kAspectRatio = 1.0 / 1.0;
+  auto input_frame = ::absl::make_unique<ImageFrame>(
+      ImageFormat::SRGB, kFixedWidth,
+      static_cast<int>(kFixedWidth / kAspectRatio), 16);
+  runner->MutableInputs()
+      ->Tag("INPUT_FRAMES")
+      .packets.push_back(Adopt(input_frame.release()).At(Timestamp(1000)));
+  ::mediapipe::Status status = runner->Run();
+  EXPECT_EQ(status.code(), ::mediapipe::StatusCode::kUnknown);
+  EXPECT_THAT(status.ToString(),
+              ::testing::HasSubstr("Failing due to aspect ratio"));
+}
+
+}  // namespace
+}  // namespace autoflip
+}  // namespace mediapipe

mediapipe/examples/desktop/autoflip/quality/BUILD

@@ -0,0 +1,331 @@
+load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
+
+# Copyright 2019 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+licenses(["notice"])  # Apache 2.0
+
+package(default_visibility = ["//mediapipe/examples:__subpackages__"])
+
+proto_library(
+    name = "cropping_proto",
+    srcs = ["cropping.proto"],
+    deps = [
+        "//mediapipe/examples/desktop/autoflip:autoflip_messages_proto",
+    ],
+)
+
+mediapipe_cc_proto_library(
+    name = "cropping_cc_proto",
+    srcs = ["cropping.proto"],
+    cc_deps = ["//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto"],
+    visibility = ["//mediapipe/examples:__subpackages__"],
+    deps = [":cropping_proto"],
+)
+
+proto_library(
+    name = "focus_point_proto",
+    srcs = ["focus_point.proto"],
+)
+
+mediapipe_cc_proto_library(
+    name = "focus_point_cc_proto",
+    srcs = ["focus_point.proto"],
+    visibility = ["//mediapipe/examples:__subpackages__"],
+    deps = [":focus_point_proto"],
+)
+
+cc_library(
+    name = "frame_crop_region_computer",
+    srcs = ["frame_crop_region_computer.cc"],
+    hdrs = ["frame_crop_region_computer.h"],
+    deps = [
+        ":cropping_cc_proto",
+        ":utils",
+        "//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+    ],
+)
+
+cc_library(
+    name = "math_utils",
+    hdrs = ["math_utils.h"],
+)
+
+cc_library(
+    name = "piecewise_linear_function",
+    srcs = ["piecewise_linear_function.cc"],
+    hdrs = ["piecewise_linear_function.h"],
+    deps = [
+        "//mediapipe/framework/port:status",
+    ],
+)
+
+cc_library(
+    name = "padding_effect_generator",
+    srcs = ["padding_effect_generator.cc"],
+    hdrs = ["padding_effect_generator.h"],
+    deps = [
+        "//mediapipe/framework/formats:image_frame",
+        "//mediapipe/framework/formats:image_frame_opencv",
+        "//mediapipe/framework/port:commandlineflags",
+        "//mediapipe/framework/port:opencv_core",
+        "//mediapipe/framework/port:opencv_imgproc",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+    ],
+)
+
+cc_library(
+    name = "scene_camera_motion_analyzer",
+    srcs = ["scene_camera_motion_analyzer.cc"],
+    hdrs = ["scene_camera_motion_analyzer.h"],
+    deps = [
+        ":cropping_cc_proto",
+        ":focus_point_cc_proto",
+        ":math_utils",
+        ":piecewise_linear_function",
+        ":utils",
+        "//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto",
+        "//mediapipe/framework:timestamp",
+        "//mediapipe/framework/port:integral_types",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/memory",
+        "@com_google_absl//absl/strings",
+        "@com_google_absl//absl/strings:str_format",
+    ],
+)
+
+cc_library(
+    name = "scene_cropping_viz",
+    srcs = ["scene_cropping_viz.cc"],
+    hdrs = ["scene_cropping_viz.h"],
+    deps = [
+        ":cropping_cc_proto",
+        ":focus_point_cc_proto",
+        "//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:image_format_cc_proto",
+        "//mediapipe/framework/formats:image_frame",
+        "//mediapipe/framework/formats:image_frame_opencv",
+        "//mediapipe/framework/port:opencv_core",
+        "//mediapipe/framework/port:opencv_imgproc",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/memory",
+    ],
+)
+
+cc_library(
+    name = "polynomial_regression_path_solver",
+    srcs = ["polynomial_regression_path_solver.cc"],
+    hdrs = ["polynomial_regression_path_solver.h"],
+    deps = [
+        ":focus_point_cc_proto",
+        "//mediapipe/framework/port:opencv_core",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "@ceres_solver//:ceres",
+    ],
+)
+
+cc_library(
+    name = "scene_cropper",
+    srcs = ["scene_cropper.cc"],
+    hdrs = ["scene_cropper.h"],
+    deps = [
+        ":cropping_cc_proto",
+        ":focus_point_cc_proto",
+        ":polynomial_regression_path_solver",
+        ":utils",
+        "//mediapipe/framework/port:opencv_core",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/memory",
+    ],
+)
+
+cc_library(
+    name = "utils",
+    srcs = ["utils.cc"],
+    hdrs = ["utils.h"],
+    deps = [
+        ":cropping_cc_proto",
+        ":math_utils",
+        ":piecewise_linear_function",
+        "//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto",
+        "//mediapipe/framework/port:opencv_core",
+        "//mediapipe/framework/port:opencv_imgproc",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/memory",
+    ],
+)
+
+cc_test(
+    name = "frame_crop_region_computer_test",
+    srcs = ["frame_crop_region_computer_test.cc"],
+    deps = [
+        ":cropping_cc_proto",
+        ":frame_crop_region_computer",
+        "//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto",
+        "//mediapipe/framework/port:gtest_main",
+        "@com_google_absl//absl/memory",
+    ],
+)
+
+cc_test(
+    name = "piecewise_linear_function_test",
+    srcs = ["piecewise_linear_function_test.cc"],
+    deps = [
+        ":piecewise_linear_function",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:status",
+    ],
+)
+
+cc_test(
+    name = "scene_camera_motion_analyzer_test",
+    srcs = ["scene_camera_motion_analyzer_test.cc"],
+    data = [
+        "//mediapipe/examples/desktop/autoflip/quality/testdata:camera_motion_tracking_scene_frame_results.csv",
+    ],
+    deps = [
+        ":focus_point_cc_proto",
+        ":piecewise_linear_function",
+        ":scene_camera_motion_analyzer",
+        "//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto",
+        "//mediapipe/framework/deps:file_path",
+        "//mediapipe/framework/port:file_helpers",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
+cc_test(
+    name = "padding_effect_generator_test",
+    srcs = ["padding_effect_generator_test.cc"],
+    data = [
+        "//mediapipe/examples/desktop/autoflip/quality/testdata:google.jpg",
+        "//mediapipe/examples/desktop/autoflip/quality/testdata:result_0.3.jpg",
+        "//mediapipe/examples/desktop/autoflip/quality/testdata:result_0.3_solid_background.jpg",
+        "//mediapipe/examples/desktop/autoflip/quality/testdata:result_0.6.jpg",
+        "//mediapipe/examples/desktop/autoflip/quality/testdata:result_0.6_solid_background.jpg",
+        "//mediapipe/examples/desktop/autoflip/quality/testdata:result_1.6.jpg",
+        "//mediapipe/examples/desktop/autoflip/quality/testdata:result_1.6_solid_background.jpg",
+        "//mediapipe/examples/desktop/autoflip/quality/testdata:result_1.jpg",
+        "//mediapipe/examples/desktop/autoflip/quality/testdata:result_1_solid_background.jpg",
+        "//mediapipe/examples/desktop/autoflip/quality/testdata:result_2.5.jpg",
+        "//mediapipe/examples/desktop/autoflip/quality/testdata:result_2.5_solid_background.jpg",
+        "//mediapipe/examples/desktop/autoflip/quality/testdata:result_3.4.jpg",
+        "//mediapipe/examples/desktop/autoflip/quality/testdata:result_3.4_solid_background.jpg",
+    ],
+    deps = [
+        ":padding_effect_generator",
+        "//mediapipe/framework/deps:file_path",
+        "//mediapipe/framework/formats:image_frame",
+        "//mediapipe/framework/formats:image_frame_opencv",
+        "//mediapipe/framework/port:commandlineflags",
+        "//mediapipe/framework/port:file_helpers",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:opencv_imgcodecs",
+        "//mediapipe/framework/port:opencv_imgproc",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
+cc_test(
+    name = "polynomial_regression_path_solver_test",
+    srcs = ["polynomial_regression_path_solver_test.cc"],
+    deps = [
+        ":focus_point_cc_proto",
+        ":polynomial_regression_path_solver",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:opencv_core",
+        "//mediapipe/framework/port:status",
+    ],
+)
+
+cc_test(
+    name = "scene_cropper_test",
+    size = "small",
+    timeout = "short",
+    srcs = ["scene_cropper_test.cc"],
+    deps = [
+        ":focus_point_cc_proto",
+        ":scene_cropper",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:opencv_core",
+    ],
+)
+
+cc_test(
+    name = "utils_test",
+    srcs = ["utils_test.cc"],
+    deps = [
+        ":cropping_cc_proto",
+        ":utils",
+        "//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:integral_types",
+        "//mediapipe/framework/port:opencv_core",
+    ],
+)
+
+proto_library(
+    name = "visual_scorer_proto",
+    srcs = ["visual_scorer.proto"],
+)
+
+mediapipe_cc_proto_library(
+    name = "visual_scorer_cc_proto",
+    srcs = ["visual_scorer.proto"],
+    visibility = ["//mediapipe/examples:__subpackages__"],
+    deps = [":visual_scorer_proto"],
+)
+
+cc_library(
+    name = "visual_scorer",
+    srcs = ["visual_scorer.cc"],
+    hdrs = ["visual_scorer.h"],
+    deps = [
+        ":visual_scorer_cc_proto",
+        "//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto",
+        "//mediapipe/framework/port:opencv_core",
+        "//mediapipe/framework/port:opencv_imgproc",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+    ],
+)
+
+cc_test(
+    name = "visual_scorer_test",
+    srcs = [
+        "visual_scorer_test.cc",
+    ],
+    linkstatic = 1,
+    deps = [
+        ":visual_scorer",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:opencv_core",
+        "//mediapipe/framework/port:opencv_imgproc",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/port:status",
+    ],
+)

mediapipe/examples/desktop/autoflip/quality/cropping.proto

@@ -0,0 +1,217 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe.autoflip;
+
+import "mediapipe/examples/desktop/autoflip/autoflip_messages.proto";
+
+// All relevant information for key frames, including timestamp and detected
+// features. This object should be generated by calling PackKeyFrameInfo() in
+// the util namespace. It is passed in to ComputeFrameCropRegion().
+message KeyFrameInfo {
+  // Frame timestamp (in microseconds).
+  optional int64 timestamp_ms = 1;
+  // Detected features.
+  optional DetectionSet detections = 2;
+}
+
+// User-specified key frame crop options (such as target width and height).
+message KeyFrameCropOptions {
+  // Target crop size.
+  // Note: if you are using the SceneCroppingCalculator, DO NOT set these fields
+  // manually as they will be then overwritten inside the calculator.
+  optional int32 target_width = 1;
+  optional int32 target_height = 2;
+  // Option for how region score is aggregated from individual feature scores.
+  // TODO: consider merging this enum type into the signal fusing
+  // calculator.
+  enum ScoreAggregationType {
+    // Unknown value (should not be used).
+    UNKNOWN = 0;
+    // Takes the score of the feature with maximum score.
+    MAXIMUM = 1;
+    // Takes the sum of the scores of the required regions.
+    SUM_REQUIRED = 2;
+    // Takes the sum of the scores of all the regions that are fully covered.
+    SUM_ALL = 3;
+    // Uses a constant score 1.0 for all crop regions.
+    CONSTANT = 4;
+  }
+  optional ScoreAggregationType score_aggregation_type = 3 [default = SUM_ALL];
+  // Minimum centered coverage fraction (in length, not area) for a non-required
+  // region to be included in the crop region. Applies to both dimensions.
+  optional float non_required_region_min_coverage_fraction = 4 [default = 0.5];
+}
+
+// Key frame crop result containing the crop region rectangle, along with
+// summary information on the cropping, such as whether all required regions
+// could fit inside the target size, and what fraction of non-required regions
+// are fully covered. This object is returned by ComputeFrameCropRegion() in
+// the FrameCropRegionComputer class.
+message KeyFrameCropResult {
+  // Successfully covers all required features. If there are no required
+  // regions, this field is set to true.
+  optional bool are_required_regions_covered_in_target_size = 1;
+  // Fraction of non-required features covered.
+  optional float fraction_non_required_covered = 2;
+  // Whether required crop region is empty (no detections).
+  optional bool required_region_is_empty = 3;
+  // Whether (full) crop region is empty (no detections).
+  optional bool region_is_empty = 4;
+  // Computed required crop region.
+  optional Rect required_region = 5;
+  // Computed (full) crop region.
+  optional Rect region = 6;
+  // Score of the computed crop region based on the detected features.
+  optional float region_score = 7;
+}
+
+// Compact processed scene key frame info containing timestamp, center position,
+// and score. Each key frame has one SceneKeyFrameCompactInfo in
+// SceneKeyFrameCropSummary.
+message SceneKeyFrameCompactInfo {
+  // Key frame timestamp (in microseconds).
+  optional int64 timestamp_ms = 1;
+  // Key frame crop region center in the horizontal/vertical directions (in
+  // pixels).
+  optional float center_x = 2;
+  optional float center_y = 3;
+  // Key frame crop region score.
+  optional float score = 4;
+}
+
+// Summary information for the key frame crop results in a scene. Computed by
+// AnalyzeSceneKeyFrameCropResults() in the SceneCameraMotionAnalyzer class.
+// Used to decide camera motion type and populate salient point frames.
+message SceneKeyFrameCropSummary {
+  // Scene frame size.
+  optional int32 scene_frame_width = 1;
+  optional int32 scene_frame_height = 2;
+
+  // Number of key frames in the scene.
+  optional int32 num_key_frames = 3;
+  // Scene key frame compact infos.
+  repeated SceneKeyFrameCompactInfo key_frame_compact_infos = 4;
+
+  // The minimum/maximum values of key frames' crop centers in the horizontal/
+  // vertical directions.
+  optional float key_frame_center_min_x = 5;
+  optional float key_frame_center_max_x = 6;
+  optional float key_frame_center_min_y = 7;
+  optional float key_frame_center_max_y = 8;
+
+  // The union of all the key frame required crop regions. When camera is steady
+  // the crop window is set to cover this union.
+  optional Rect key_frame_required_crop_region_union = 9;
+
+  // The minimum/maximum scores of key frames' crop regions.
+  optional float key_frame_min_score = 10;
+  optional float key_frame_max_score = 11;
+
+  // Size of the scene's crop window, calculated as the maximum of the target
+  // size and the largest size of the key frames' crop regions in the scene.
+  optional int32 crop_window_width = 12;
+  optional int32 crop_window_height = 13;
+
+  // Indicator for whether the scene has any frame with any salient region.
+  optional bool has_salient_region = 14;
+  // Indicator for whether the scene has any frame with any required salient
+  // region.
+  optional bool has_required_salient_region = 15;
+  // Percentage of key frames that are successfully cropped (i.e. covers all
+  // required regions inside the target size).
+  optional float frame_success_rate = 16;
+  // Amount of motion in the horizontal/vertical direction (i.e. the horizontal/
+  // vertical range of the key frame crop centers' position as a fraction of
+  // frame width/height).
+  optional float horizontal_motion_amount = 17;
+  optional float vertical_motion_amount = 18;
+}
+
+// Scene camera motion determined by the SceneCameraMotionAnalyzer class.
+message SceneCameraMotion {
+  // Camera focuses on a fixed center throughout the scene.
+  message SteadyMotion {
+    // Steady look-at center in horizontal/vertical directions (in pixels).
+    optional float steady_look_at_center_x = 1;
+    optional float steady_look_at_center_y = 2;
+  }
+  // Camera tracks key frame salient region centers.
+  message TrackingMotion {
+    // Fields to be added if necessary.
+  }
+  // Camera sweeps from one point to another.
+  message SweepingMotion {
+    // Starting and ending center positions for camera sweeping in pixels.
+    optional float sweep_start_center_x = 1;
+    optional float sweep_start_center_y = 2;
+    optional float sweep_end_center_x = 3;
+    optional float sweep_end_center_y = 4;
+  }
+  oneof motion_type {
+    SteadyMotion steady_motion = 1;
+    TrackingMotion tracking_motion = 2;
+    SweepingMotion sweeping_motion = 3;
+    // Other types that we might support later.
+  }
+}
+
+// User-specified options for analyzing scene camera motion from a collection of
+// key frame crop regions.
+message SceneCameraMotionAnalyzerOptions {
+  // If there is small motion within the scene keep the camera steady at the
+  // center.
+  optional float motion_stabilization_threshold_percent = 1 [default = .30];
+  // Snap to center if there is small motion and already focused closed to the
+  // center.
+  optional float snap_center_max_distance_percent = 2 [default = .08];
+  // Maximum weight for a constraint. Scales scores accordingly so that the
+  // maximum score is equal to this weight.
+  optional float maximum_salient_point_weight = 3 [default = 100.0];
+  // Normalized bound for SalientPoint's in the frame from the border. This is
+  // uniformly applied to the left, right, top, and bottom. It should be
+  // strictly less than 0.5. A narrower bound (closer to 0.5) gives better
+  // constraint enforcement.
+  optional float salient_point_bound = 4 [default = 0.48];
+  // Indicator for whether sweeping is allowed. Note that if a scene can be
+  // seamlessly padded with solid background color, sweeping will be disabled
+  // regardlessly of the value of this flag.
+  optional bool allow_sweeping = 5 [default = true];
+  // Minimal scene time span in seconds to allow camera sweeping.
+  optional float minimum_scene_span_sec_for_sweeping = 6 [default = 1.0];
+  // If success rate in a scene is less than this, then use camera sweeping.
+  optional float minimum_success_rate_for_sweeping = 7 [default = 0.4];
+  // If true, sweep entire frame. Otherwise, sweep the crop window.
+  optional bool sweep_entire_frame = 8 [default = true];
+}
+
+// Video cropping summary information for debugging/statistics.
+message VideoCroppingSummary {
+  message SceneCroppingSummary {
+    // Scene span in seconds.
+    optional float start_sec = 1;
+    optional float end_sec = 2;
+    // Indicator for whether this scene was cut at a real physical scene
+    // boundary (as opposed to force flush).
+    optional bool is_end_of_scene = 3;
+    // Scene camera motion.
+    optional SceneCameraMotion camera_motion = 4;
+    // Indicator for whether the scene is padded.
+    optional bool is_padded = 5;
+  }
+  // Cropping summaries for all the scenes in the video.
+  repeated SceneCroppingSummary scene_summaries = 1;
+}

mediapipe/examples/desktop/autoflip/quality/focus_point.proto

@@ -0,0 +1,79 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe.autoflip;
+
+// Focus point location (normalized w.r.t. frame_width and frame_height, i.e.
+// specified in the domain [0, 1] x [0, 1]).
+
+// For TYPE_INCLUDE:
+// During retargeting and stabilization focus points introduce constraints
+// that will try to keep the normalized location in the rectangle
+// frame_size - normalized bounds.
+// For this soft constraints are used, therefore the weight specifies
+// how "important" the focus point is (higher is better).
+// In particular for each point p the retargeter introduces two pairs of
+// constraints of the form:
+//           x - slack < width - right
+//    and    x + slack > 0 + left,        with slack > 0
+//  where the weight specifies the importance of the slack.
+//
+// For TYPE_EXCLUDE_*:
+// Similar to above, but constraints are introduced to keep
+// the point to the left of the left bound OR the right of the right bound.
+// In particular:
+//          x - slack < left OR
+//          x + slack >= right
+//  Similar to above, the weight specifies the importance of the slack.
+//
+// Note: Choosing a too high weight can lead to
+// jerkiness as the stabilization essentially starts tracking the focus point.
+message FocusPoint {
+  // Normalized location of the point (within domain [0, 1] x [0, 1].
+  optional float norm_point_x = 1 [default = 0.0];
+  optional float norm_point_y = 2 [default = 0.0];
+
+  enum FocusPointType {
+    TYPE_INCLUDE = 1;
+    TYPE_EXCLUDE_LEFT = 2;
+    TYPE_EXCLUDE_RIGHT = 3;
+  }
+
+  // Focus point type. By default we try to frame the focus point within
+  // the bounding box specified by left, bottom, right, top. Alternatively, one
+  // can choose to exclude the point. For details, see discussion above.
+  optional FocusPointType type = 11 [default = TYPE_INCLUDE];
+
+  // Bounds are specified in normalized coordinates [0, 1], FROM the specified
+  // border. Opposing bounds (e.g. left and right) may not add to values
+  // larger than 1.
+  // Default bounds center focus point within centering third of the frame.
+  optional float left = 3 [default = 0.3];
+  optional float bottom = 4 [default = 0.3];
+  optional float right = 9 [default = 0.3];
+  optional float top = 10 [default = 0.3];
+
+  optional float weight = 5 [default = 15];
+
+  extensions 20000 to max;
+}
+
+// Aggregates FocusPoint's for a frame.
+message FocusPointFrame {
+  repeated FocusPoint point = 1;
+
+  extensions 20000 to max;
+}

mediapipe/examples/desktop/autoflip/quality/frame_crop_region_computer.cc

@@ -0,0 +1,259 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/examples/desktop/autoflip/quality/frame_crop_region_computer.h"
+
+#include <cmath>
+
+#include "mediapipe/examples/desktop/autoflip/quality/utils.h"
+#include "mediapipe/framework/port/ret_check.h"
+
+namespace mediapipe {
+namespace autoflip {
+
+::mediapipe::Status FrameCropRegionComputer::ExpandSegmentUnderConstraint(
+    const Segment& segment_to_add, const Segment& base_segment,
+    const int max_length, Segment* combined_segment,
+    CoverType* cover_type) const {
+  RET_CHECK(combined_segment != nullptr) << "Combined segment is null.";
+  RET_CHECK(cover_type != nullptr) << "Cover type is null.";
+
+  const LeftPoint segment_to_add_left = segment_to_add.first;
+  const RightPoint segment_to_add_right = segment_to_add.second;
+  RET_CHECK(segment_to_add_right >= segment_to_add_left)
+      << "Invalid segment to add.";
+  const LeftPoint base_segment_left = base_segment.first;
+  const RightPoint base_segment_right = base_segment.second;
+  RET_CHECK(base_segment_right >= base_segment_left) << "Invalid base segment.";
+  const int base_length = base_segment_right - base_segment_left;
+  RET_CHECK(base_length <= max_length)
+      << "Base segment length exceeds max length.";
+
+  const int segment_to_add_length = segment_to_add_right - segment_to_add_left;
+  const int max_leftout_amount =
+      std::ceil((1.0 - options_.non_required_region_min_coverage_fraction()) *
+                segment_to_add_length / 2);
+  const LeftPoint min_coverage_segment_to_add_left =
+      segment_to_add_left + max_leftout_amount;
+  const LeftPoint min_coverage_segment_to_add_right =
+      segment_to_add_right - max_leftout_amount;
+
+  LeftPoint combined_segment_left =
+      std::min(segment_to_add_left, base_segment_left);
+  RightPoint combined_segment_right =
+      std::max(segment_to_add_right, base_segment_right);
+
+  LeftPoint min_coverage_combined_segment_left =
+      std::min(min_coverage_segment_to_add_left, base_segment_left);
+  RightPoint min_coverage_combined_segment_right =
+      std::max(min_coverage_segment_to_add_right, base_segment_right);
+
+  if ((combined_segment_right - combined_segment_left) <= max_length) {
+    *cover_type = FULLY_COVERED;
+  } else if (min_coverage_combined_segment_right -
+                 min_coverage_combined_segment_left <=
+             max_length) {
+    *cover_type = PARTIALLY_COVERED;
+    combined_segment_left = min_coverage_combined_segment_left;
+    combined_segment_right = min_coverage_combined_segment_right;
+  } else {
+    *cover_type = NOT_COVERED;
+    combined_segment_left = base_segment_left;
+    combined_segment_right = base_segment_right;
+  }
+
+  *combined_segment =
+      std::make_pair(combined_segment_left, combined_segment_right);
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status FrameCropRegionComputer::ExpandRectUnderConstraints(
+    const Rect& rect_to_add, const int max_width, const int max_height,
+    Rect* base_rect, CoverType* cover_type) const {
+  RET_CHECK(base_rect != nullptr) << "Base rect is null.";
+  RET_CHECK(cover_type != nullptr) << "Cover type is null.";
+  RET_CHECK(base_rect->width() <= max_width &&
+            base_rect->height() <= max_height)
+      << "Base rect already exceeds target size.";
+
+  const LeftPoint rect_to_add_left = rect_to_add.x();
+  const RightPoint rect_to_add_right = rect_to_add.x() + rect_to_add.width();
+  const LeftPoint rect_to_add_top = rect_to_add.y();
+  const RightPoint rect_to_add_bottom = rect_to_add.y() + rect_to_add.height();
+  const LeftPoint base_rect_left = base_rect->x();
+  const RightPoint base_rect_right = base_rect->x() + base_rect->width();
+  const LeftPoint base_rect_top = base_rect->y();
+  const RightPoint base_rect_bottom = base_rect->y() + base_rect->height();
+
+  Segment horizontal_combined_segment, vertical_combined_segment;
+  CoverType horizontal_cover_type, vertical_cover_type;
+  const auto horizontal_status = ExpandSegmentUnderConstraint(
+      std::make_pair(rect_to_add_left, rect_to_add_right),
+      std::make_pair(base_rect_left, base_rect_right), max_width,
+      &horizontal_combined_segment, &horizontal_cover_type);
+  MP_RETURN_IF_ERROR(horizontal_status);
+  const auto vertical_status = ExpandSegmentUnderConstraint(
+      std::make_pair(rect_to_add_top, rect_to_add_bottom),
+      std::make_pair(base_rect_top, base_rect_bottom), max_height,
+      &vertical_combined_segment, &vertical_cover_type);
+  MP_RETURN_IF_ERROR(vertical_status);
+
+  if (horizontal_cover_type == NOT_COVERED ||
+      vertical_cover_type == NOT_COVERED) {
+    // Gives up if the segment is not covered in either direction.
+    *cover_type = NOT_COVERED;
+  } else {
+    // Tries to (partially) cover the new rect to be added.
+    base_rect->set_x(horizontal_combined_segment.first);
+    base_rect->set_y(vertical_combined_segment.first);
+    base_rect->set_width(horizontal_combined_segment.second -
+                         horizontal_combined_segment.first);
+    base_rect->set_height(vertical_combined_segment.second -
+                          vertical_combined_segment.first);
+    if (horizontal_cover_type == FULLY_COVERED &&
+        vertical_cover_type == FULLY_COVERED) {
+      *cover_type = FULLY_COVERED;
+    } else {
+      *cover_type = PARTIALLY_COVERED;
+    }
+  }
+
+  return ::mediapipe::OkStatus();
+}
+
+void FrameCropRegionComputer::UpdateCropRegionScore(
+    const KeyFrameCropOptions::ScoreAggregationType score_aggregation_type,
+    const float feature_score, const bool is_required,
+    float* crop_region_score) {
+  if (feature_score < 0.0) {
+    LOG(WARNING) << "Ignoring negative score";
+    return;
+  }
+
+  switch (score_aggregation_type) {
+    case KeyFrameCropOptions::MAXIMUM: {
+      *crop_region_score = std::max(feature_score, *crop_region_score);
+      break;
+    }
+    case KeyFrameCropOptions::SUM_REQUIRED: {
+      if (is_required) {
+        *crop_region_score += feature_score;
+      }
+      break;
+    }
+    case KeyFrameCropOptions::SUM_ALL: {
+      *crop_region_score += feature_score;
+      break;
+    }
+    case KeyFrameCropOptions::CONSTANT: {
+      *crop_region_score = 1.0;
+      break;
+    }
+    default: {
+      LOG(WARNING) << "Unknown CropRegionScoreType " << score_aggregation_type;
+      break;
+    }
+  }
+}
+
+::mediapipe::Status FrameCropRegionComputer::ComputeFrameCropRegion(
+    const KeyFrameInfo& frame_info, KeyFrameCropResult* crop_result) const {
+  RET_CHECK(crop_result != nullptr) << "KeyFrameCropResult is null.";
+
+  // Sorts required and non-required regions.
+  std::vector<SalientRegion> required_regions, non_required_regions;
+  const auto sort_status = SortDetections(
+      frame_info.detections(), &required_regions, &non_required_regions);
+  MP_RETURN_IF_ERROR(sort_status);
+
+  int target_width = options_.target_width();
+  int target_height = options_.target_height();
+  auto* region = crop_result->mutable_region();
+  RET_CHECK(region != nullptr) << "Crop region is null.";
+
+  bool crop_region_is_empty = true;
+  float crop_region_score = 0.0;
+
+  // Gets union of all required regions.
+  for (int i = 0; i < required_regions.size(); ++i) {
+    const Rect& required_region = required_regions[i].location();
+    if (crop_region_is_empty) {
+      *region = required_region;
+      crop_region_is_empty = false;
+    } else {
+      RectUnion(required_region, region);
+    }
+    UpdateCropRegionScore(options_.score_aggregation_type(),
+                          required_regions[i].score(), true,
+                          &crop_region_score);
+  }
+  crop_result->set_required_region_is_empty(crop_region_is_empty);
+  if (!crop_region_is_empty) {
+    *crop_result->mutable_required_region() = *region;
+    crop_result->set_are_required_regions_covered_in_target_size(
+        region->width() <= target_width && region->height() <= target_height);
+    target_width = std::max(target_width, region->width());
+    target_height = std::max(target_height, region->height());
+  } else {
+    crop_result->set_are_required_regions_covered_in_target_size(true);
+  }
+
+  // Tries to fit non-required regions.
+  int num_covered = 0;
+  for (int i = 0; i < non_required_regions.size(); ++i) {
+    const Rect& non_required_region = non_required_regions[i].location();
+    CoverType cover_type = NOT_COVERED;
+    if (crop_region_is_empty) {
+      // If the crop region is empty, tries to expand an empty base region
+      // at the center of this region to include itself.
+      region->set_x(non_required_region.x() + non_required_region.width() / 2);
+      region->set_y(non_required_region.y() + non_required_region.height() / 2);
+      region->set_width(0);
+      region->set_height(0);
+      MP_RETURN_IF_ERROR(ExpandRectUnderConstraints(non_required_region,
+                                                    target_width, target_height,
+                                                    region, &cover_type));
+      if (cover_type != NOT_COVERED) {
+        crop_region_is_empty = false;
+      }
+    } else {
+      // Otherwise tries to expand the crop region to cover the non-required
+      // region under target size constraint.
+      MP_RETURN_IF_ERROR(ExpandRectUnderConstraints(non_required_region,
+                                                    target_width, target_height,
+                                                    region, &cover_type));
+    }
+
+    // Updates number of covered non-required regions and score.
+    if (cover_type == FULLY_COVERED) {
+      num_covered++;
+      UpdateCropRegionScore(options_.score_aggregation_type(),
+                            non_required_regions[i].score(), false,
+                            &crop_region_score);
+    }
+  }
+
+  const float fraction_covered =
+      non_required_regions.empty()
+          ? 0.0
+          : static_cast<float>(num_covered) / non_required_regions.size();
+  crop_result->set_fraction_non_required_covered(fraction_covered);
+
+  crop_result->set_region_is_empty(crop_region_is_empty);
+  crop_result->set_region_score(crop_region_score);
+  return ::mediapipe::OkStatus();
+}
+
+}  // namespace autoflip
+}  // namespace mediapipe

mediapipe/examples/desktop/autoflip/quality/frame_crop_region_computer.h

@@ -0,0 +1,110 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_FRAME_CROP_REGION_COMPUTER_H_
+#define MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_FRAME_CROP_REGION_COMPUTER_H_
+
+#include <vector>
+
+#include "mediapipe/examples/desktop/autoflip/autoflip_messages.pb.h"
+#include "mediapipe/examples/desktop/autoflip/quality/cropping.pb.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+namespace autoflip {
+
+// This class computes per-frame crop regions based on crop frame options.
+// It aggregates required regions and then tries to fit in non-required regions
+// with best effort. It does not make use of static features.
+class FrameCropRegionComputer {
+ public:
+  FrameCropRegionComputer() = delete;
+
+  explicit FrameCropRegionComputer(
+      const KeyFrameCropOptions& crop_frame_options)
+      : options_(crop_frame_options) {}
+
+  ~FrameCropRegionComputer() {}
+
+  // Computes the crop region for the key frame using the crop options. The crop
+  // region covers all the required regions, and attempts to cover the
+  // non-required regions with best effort. Note: this function does not
+  // consider static features, and simply tries to fit the detected features
+  // within the target frame size. The score of the crop region is aggregated
+  // from individual feature scores given the score aggregation type.
+  ::mediapipe::Status ComputeFrameCropRegion(
+      const KeyFrameInfo& frame_info, KeyFrameCropResult* crop_result) const;
+
+ protected:
+  // A segment is a 1-d object defined by its left and right point.
+  using LeftPoint = int;
+  using RightPoint = int;
+  using Segment = std::pair<LeftPoint, RightPoint>;
+  // How much a segment is covered in the combined segment.
+  enum CoverType {
+    FULLY_COVERED = 1,
+    PARTIALLY_COVERED = 2,
+    NOT_COVERED = 3,
+  };
+  // Expands a base segment to cover a segment to be added given maximum length
+  // constraint. The operation is best-effort. The resulting enlarged segment is
+  // set in the returned combined segment. Returns a CoverType to indicate the
+  // coverage of the segment to be added in the combined segment.
+  // There are 3 cases:
+  //   case 1: the length of the union of the two segments is not larger than
+  //           the maximum length.
+  //           In this case the combined segment is simply the union, and cover
+  //           type is FULLY_COVERED.
+  //   case 2: the union of the two segments exceeds the maximum length, but the
+  //           union of the base segment and required minimum centered fraction
+  //           of the new segment fits in the maximum length.
+  //           In this case the combined segment is this latter union, and cover
+  //           type is PARTIALLY_COVERED.
+  //   case 3: the union of the base segment and required minimum centered
+  //           fraction of the new segment exceeds the maximum length.
+  //           In this case the combined segment is the base segment, and cover
+  //           type is NOT_COVERED.
+  ::mediapipe::Status ExpandSegmentUnderConstraint(
+      const Segment& segment_to_add, const Segment& base_segment,
+      const int max_length, Segment* combined_segment,
+      CoverType* cover_type) const;
+
+  // Expands a base rectangle to cover a new rectangle to be added under width
+  // and height constraints. The operation is best-effort. It considers
+  // horizontal and vertical directions separately, using the
+  // ExpandSegmentUnderConstraint function for each direction. The cover type is
+  // FULLY_COVERED if the new rectangle is fully covered in both directions,
+  // PARTIALLY_COVERED if it is at least partially covered in both directions,
+  // and NOT_COVERED if it is not covered in either direction.
+  ::mediapipe::Status ExpandRectUnderConstraints(const Rect& rect_to_add,
+                                                 const int max_width,
+                                                 const int max_height,
+                                                 Rect* base_rect,
+                                                 CoverType* cover_type) const;
+
+  // Updates crop region score given current feature score, whether the feature
+  // is required, and the score aggregation type. Ignores negative scores.
+  static void UpdateCropRegionScore(
+      const KeyFrameCropOptions::ScoreAggregationType score_aggregation_type,
+      const float feature_score, const bool is_required,
+      float* crop_region_score);
+
+ private:
+  // Crop frame options.
+  KeyFrameCropOptions options_;
+};
+}  // namespace autoflip
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_FRAME_CROP_REGION_COMPUTER_H_

mediapipe/examples/desktop/autoflip/quality/frame_crop_region_computer_test.cc

@@ -0,0 +1,579 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/examples/desktop/autoflip/quality/frame_crop_region_computer.h"
+
+#include <memory>
+
+#include "absl/memory/memory.h"
+#include "mediapipe/examples/desktop/autoflip/autoflip_messages.pb.h"
+#include "mediapipe/examples/desktop/autoflip/quality/cropping.pb.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/status_matchers.h"
+
+namespace mediapipe {
+namespace autoflip {
+
+using ::testing::HasSubstr;
+
+const int kSegmentMaxLength = 10;
+const int kTargetWidth = 500;
+const int kTargetHeight = 1000;
+
+// Makes a rectangle given the corner (x, y) and the size (width, height).
+Rect MakeRect(const int x, const int y, const int width, const int height) {
+  Rect rect;
+  rect.set_x(x);
+  rect.set_y(y);
+  rect.set_width(width);
+  rect.set_height(height);
+  return rect;
+}
+
+// Adds a detection to the key frame info given its location, whether it is
+// required, and its score. The score is default to 1.0.
+void AddDetection(const Rect& rect, const bool is_required,
+                  KeyFrameInfo* key_frame_info, const float score = 1.0) {
+  auto* detection = key_frame_info->mutable_detections()->add_detections();
+  *(detection->mutable_location()) = rect;
+  detection->set_score(score);
+  detection->set_is_required(is_required);
+}
+
+// Makes key frame crop options given target width and height.
+KeyFrameCropOptions MakeKeyFrameCropOptions(const int target_width,
+                                            const int target_height) {
+  KeyFrameCropOptions options;
+  options.set_target_width(target_width);
+  options.set_target_height(target_height);
+  return options;
+}
+
+// Checks whether rectangle a is inside rectangle b.
+bool CheckRectIsInside(const Rect& rect_a, const Rect& rect_b) {
+  return (rect_b.x() <= rect_a.x() && rect_b.y() <= rect_a.y() &&
+          rect_b.x() + rect_b.width() >= rect_a.x() + rect_a.width() &&
+          rect_b.y() + rect_b.height() >= rect_a.y() + rect_a.height());
+}
+
+// Checks whether two rectangles are equal.
+bool CheckRectsEqual(const Rect& rect1, const Rect& rect2) {
+  return (rect1.x() == rect2.x() && rect1.y() == rect2.y() &&
+          rect1.width() == rect2.width() && rect1.height() == rect2.height());
+}
+
+// Checks whether two rectangles have non-zero overlapping area.
+bool CheckRectsOverlap(const Rect& rect1, const Rect& rect2) {
+  const int x1_left = rect1.x(), x1_right = rect1.x() + rect1.width();
+  const int y1_top = rect1.y(), y1_bottom = rect1.y() + rect1.height();
+  const int x2_left = rect2.x(), x2_right = rect2.x() + rect2.width();
+  const int y2_top = rect2.y(), y2_bottom = rect2.y() + rect2.height();
+  const int x_left = std::max(x1_left, x2_left);
+  const int x_right = std::min(x1_right, x2_right);
+  const int y_top = std::max(y1_top, y2_top);
+  const int y_bottom = std::min(y1_bottom, y2_bottom);
+  return (x_right > x_left && y_bottom > y_top);
+}
+
+// Checks that all the required regions in the detections in KeyFrameInfo are
+// covered in the KeyFrameCropResult.
+void CheckRequiredRegionsAreCovered(const KeyFrameInfo& key_frame_info,
+                                    const KeyFrameCropResult& result) {
+  bool has_required = false;
+  for (int i = 0; i < key_frame_info.detections().detections_size(); ++i) {
+    const auto& detection = key_frame_info.detections().detections(i);
+    if (detection.is_required()) {
+      has_required = true;
+      EXPECT_TRUE(
+          CheckRectIsInside(detection.location(), result.required_region()));
+    }
+  }
+  EXPECT_EQ(has_required, !result.required_region_is_empty());
+  if (has_required) {
+    EXPECT_FALSE(result.region_is_empty());
+    EXPECT_TRUE(CheckRectIsInside(result.required_region(), result.region()));
+  }
+}
+
+// Testable class that can access protected types and methods in the class.
+class TestableFrameCropRegionComputer : public FrameCropRegionComputer {
+ public:
+  explicit TestableFrameCropRegionComputer(const KeyFrameCropOptions& options)
+      : FrameCropRegionComputer(options) {}
+  using FrameCropRegionComputer::CoverType;
+  using FrameCropRegionComputer::ExpandRectUnderConstraints;
+  using FrameCropRegionComputer::ExpandSegmentUnderConstraint;
+  using FrameCropRegionComputer::FULLY_COVERED;
+  using FrameCropRegionComputer::LeftPoint;  // int
+  using FrameCropRegionComputer::NOT_COVERED;
+  using FrameCropRegionComputer::PARTIALLY_COVERED;
+  using FrameCropRegionComputer::RightPoint;  // int
+  using FrameCropRegionComputer::Segment;     // std::pair<int, int>
+  using FrameCropRegionComputer::UpdateCropRegionScore;
+
+  // Makes a segment from two endpoints.
+  static Segment MakeSegment(const LeftPoint left, const RightPoint right) {
+    return std::make_pair(left, right);
+  }
+
+  // Checks that two segments are equal.
+  static bool CheckSegmentsEqual(const Segment& segment1,
+                                 const Segment& segment2) {
+    return (segment1.first == segment2.first &&
+            segment1.second == segment2.second);
+  }
+};
+using TestClass = TestableFrameCropRegionComputer;
+
+// Returns an instance of the testable class given
+// non_required_region_min_coverage_fraction.
+std::unique_ptr<TestClass> GetTestableClass(
+    const float non_required_region_min_coverage_fraction = 0.5) {
+  KeyFrameCropOptions options;
+  options.set_non_required_region_min_coverage_fraction(
+      non_required_region_min_coverage_fraction);
+  auto test_class = absl::make_unique<TestClass>(options);
+  return test_class;
+}
+
+// Checks that ExpandSegmentUnderConstraint checks output pointers are not null.
+TEST(FrameCropRegionComputerTest, ExpandSegmentUnderConstraintCheckNull) {
+  auto test_class = GetTestableClass();
+  TestClass::CoverType cover_type;
+  TestClass::Segment base_segment = TestClass::MakeSegment(10, 15);
+  TestClass::Segment segment_to_add = TestClass::MakeSegment(5, 8);
+  TestClass::Segment combined_segment;
+  // Combined segment is null.
+  auto status = test_class->ExpandSegmentUnderConstraint(
+      segment_to_add, base_segment, kSegmentMaxLength, nullptr, &cover_type);
+  EXPECT_FALSE(status.ok());
+  EXPECT_THAT(status.ToString(), HasSubstr("Combined segment is null."));
+  // Cover type is null.
+  status = test_class->ExpandSegmentUnderConstraint(
+      segment_to_add, base_segment, kSegmentMaxLength, &combined_segment,
+      nullptr);
+  EXPECT_FALSE(status.ok());
+  EXPECT_THAT(status.ToString(), HasSubstr("Cover type is null."));
+}
+
+// Checks that ExpandSegmentUnderConstraint checks input segments are valid.
+TEST(FrameCropRegionComputerTest, ExpandSegmentUnderConstraintCheckValid) {
+  auto test_class = GetTestableClass();
+  TestClass::CoverType cover_type;
+  TestClass::Segment combined_segment;
+
+  // Invalid base segment.
+  TestClass::Segment base_segment = TestClass::MakeSegment(15, 10);
+  TestClass::Segment segment_to_add = TestClass::MakeSegment(5, 8);
+  auto status = test_class->ExpandSegmentUnderConstraint(
+      segment_to_add, base_segment, kSegmentMaxLength, &combined_segment,
+      &cover_type);
+  EXPECT_FALSE(status.ok());
+  EXPECT_THAT(status.ToString(), HasSubstr("Invalid base segment."));
+
+  // Invalid segment to add.
+  base_segment = TestClass::MakeSegment(10, 15);
+  segment_to_add = TestClass::MakeSegment(8, 5);
+  status = test_class->ExpandSegmentUnderConstraint(
+      segment_to_add, base_segment, kSegmentMaxLength, &combined_segment,
+      &cover_type);
+  EXPECT_FALSE(status.ok());
+  EXPECT_THAT(status.ToString(), HasSubstr("Invalid segment to add."));
+
+  // Base segment exceeds max length.
+  base_segment = TestClass::MakeSegment(10, 100);
+  segment_to_add = TestClass::MakeSegment(5, 8);
+  status = test_class->ExpandSegmentUnderConstraint(
+      segment_to_add, base_segment, kSegmentMaxLength, &combined_segment,
+      &cover_type);
+  EXPECT_FALSE(status.ok());
+  EXPECT_THAT(status.ToString(),
+              HasSubstr("Base segment length exceeds max length."));
+}
+
+// Checks that ExpandSegmentUnderConstraint handles case 1 properly: the length
+// of the union of the two segments is not larger than the maximum length.
+TEST(FrameCropRegionComputerTest, ExpandSegmentUnderConstraintCase1) {
+  auto test_class = GetTestableClass();
+  TestClass::Segment combined_segment;
+  TestClass::CoverType cover_type;
+  TestClass::Segment base_segment = TestClass::MakeSegment(5, 10);
+  TestClass::Segment segment_to_add = TestClass::MakeSegment(3, 8);
+  MP_EXPECT_OK(test_class->ExpandSegmentUnderConstraint(
+      segment_to_add, base_segment, kSegmentMaxLength, &combined_segment,
+      &cover_type));
+  EXPECT_EQ(cover_type, TestClass::FULLY_COVERED);
+  EXPECT_TRUE(TestClass::CheckSegmentsEqual(combined_segment,
+                                            TestClass::MakeSegment(3, 10)));
+}
+
+// Checks that ExpandSegmentUnderConstraint handles case 2 properly: the union
+// of the two segments exceeds the maximum length, but the union of the base
+// segment with the minimum coverage fraction of the new segment is within the
+// maximum length.
+TEST(FrameCropRegionComputerTest, ExpandSegmentUnderConstraintCase2) {
+  TestClass::Segment combined_segment;
+  TestClass::CoverType cover_type;
+  TestClass::Segment base_segment = TestClass::MakeSegment(4, 8);
+  TestClass::Segment segment_to_add = TestClass::MakeSegment(0, 16);
+  auto test_class = GetTestableClass();
+  MP_EXPECT_OK(test_class->ExpandSegmentUnderConstraint(
+      segment_to_add, base_segment, kSegmentMaxLength, &combined_segment,
+      &cover_type));
+  EXPECT_EQ(cover_type, TestClass::PARTIALLY_COVERED);
+  EXPECT_TRUE(TestClass::CheckSegmentsEqual(combined_segment,
+                                            TestClass::MakeSegment(4, 12)));
+}
+
+// Checks that ExpandSegmentUnderConstraint handles case 3 properly: the union
+// of the base segment with the minimum coverage fraction of the new segment
+// exceeds the maximum length.
+TEST(FrameCropRegionComputerTest, ExpandSegmentUnderConstraintCase3) {
+  TestClass::Segment combined_segment;
+  TestClass::CoverType cover_type;
+  auto test_class = GetTestableClass();
+  TestClass::Segment base_segment = TestClass::MakeSegment(6, 14);
+  TestClass::Segment segment_to_add = TestClass::MakeSegment(0, 4);
+  MP_EXPECT_OK(test_class->ExpandSegmentUnderConstraint(
+      segment_to_add, base_segment, kSegmentMaxLength, &combined_segment,
+      &cover_type));
+  EXPECT_EQ(cover_type, TestClass::NOT_COVERED);
+  EXPECT_TRUE(TestClass::CheckSegmentsEqual(combined_segment, base_segment));
+}
+
+// Checks that ExpandRectUnderConstraints checks output pointers are not null.
+TEST(FrameCropRegionComputerTest, ExpandRectUnderConstraintsChecksNotNull) {
+  auto test_class = GetTestableClass();
+  TestClass::CoverType cover_type;
+  Rect base_rect, rect_to_add;
+  // Base rect is null.
+  auto status = test_class->ExpandRectUnderConstraints(
+      rect_to_add, kTargetWidth, kTargetHeight, nullptr, &cover_type);
+  EXPECT_FALSE(status.ok());
+  EXPECT_THAT(status.ToString(), HasSubstr("Base rect is null."));
+  // Cover type is null.
+  status = test_class->ExpandRectUnderConstraints(
+      rect_to_add, kTargetWidth, kTargetHeight, &base_rect, nullptr);
+  EXPECT_FALSE(status.ok());
+  EXPECT_THAT(status.ToString(), HasSubstr("Cover type is null."));
+}
+
+// Checks that ExpandRectUnderConstraints checks base rect is valid.
+TEST(FrameCropRegionComputerTest, ExpandRectUnderConstraintsChecksBaseValid) {
+  auto test_class = GetTestableClass();
+  TestClass::CoverType cover_type;
+  Rect base_rect = MakeRect(0, 0, 2 * kTargetWidth, 2 * kTargetHeight);
+  Rect rect_to_add;
+  const auto status = test_class->ExpandRectUnderConstraints(
+      rect_to_add, kTargetWidth, kTargetHeight, &base_rect, &cover_type);
+  EXPECT_FALSE(status.ok());
+  EXPECT_THAT(status.ToString(),
+              HasSubstr("Base rect already exceeds target size."));
+}
+
+// Checks that ExpandRectUnderConstraints properly handles the case where the
+// rectangle to be added can be fully covered.
+TEST(FrameCropRegionComputerTest, ExpandRectUnderConstraintsFullyCovered) {
+  auto test_class = GetTestableClass();
+  TestClass::CoverType cover_type;
+  Rect base_rect = MakeRect(0, 0, 50, 50);
+  Rect rect_to_add = MakeRect(30, 30, 30, 30);
+  MP_EXPECT_OK(test_class->ExpandRectUnderConstraints(
+      rect_to_add, kTargetWidth, kTargetHeight, &base_rect, &cover_type));
+  EXPECT_EQ(cover_type, TestClass::FULLY_COVERED);
+  EXPECT_TRUE(CheckRectsEqual(base_rect, MakeRect(0, 0, 60, 60)));
+}
+
+// Checks that ExpandRectUnderConstraints properly handles the case where the
+// rectangle to be added can be partially covered.
+TEST(FrameCropRegionComputerTest, ExpandRectUnderConstraintsPartiallyCovered) {
+  auto test_class = GetTestableClass();
+  TestClass::CoverType cover_type;
+  // Rectangle to be added can be partially covered in both both dimensions.
+  Rect base_rect = MakeRect(0, 0, 500, 500);
+  Rect rect_to_add = MakeRect(0, 300, 600, 900);
+  MP_EXPECT_OK(test_class->ExpandRectUnderConstraints(
+      rect_to_add, kTargetWidth, kTargetHeight, &base_rect, &cover_type));
+  EXPECT_EQ(cover_type, TestClass::PARTIALLY_COVERED);
+  EXPECT_TRUE(CheckRectsEqual(base_rect, MakeRect(0, 0, 500, 975)));
+
+  // Rectangle to be added can be fully covered in one dimension and partially
+  // covered in the other dimension.
+  base_rect = MakeRect(0, 0, 400, 500);
+  rect_to_add = MakeRect(100, 300, 400, 900);
+  MP_EXPECT_OK(test_class->ExpandRectUnderConstraints(
+      rect_to_add, kTargetWidth, kTargetHeight, &base_rect, &cover_type));
+  EXPECT_EQ(cover_type, TestClass::PARTIALLY_COVERED);
+  EXPECT_TRUE(CheckRectsEqual(base_rect, MakeRect(0, 0, 500, 975)));
+}
+
+// Checks that ExpandRectUnderConstraints properly handles the case where the
+// rectangle to be added cannot be covered.
+TEST(FrameCropRegionComputerTest, ExpandRectUnderConstraintsNotCovered) {
+  TestClass::CoverType cover_type;
+  auto test_class = GetTestableClass();
+  Rect base_rect = MakeRect(0, 0, 500, 500);
+  Rect rect_to_add = MakeRect(550, 300, 100, 900);
+  MP_EXPECT_OK(test_class->ExpandRectUnderConstraints(
+      rect_to_add, kTargetWidth, kTargetHeight, &base_rect, &cover_type));
+  EXPECT_EQ(cover_type, TestClass::NOT_COVERED);  // no overlap in x dimension
+  EXPECT_TRUE(CheckRectsEqual(base_rect, MakeRect(0, 0, 500, 500)));
+}
+
+// Checks that ComputeFrameCropRegion handles the case of empty detections.
+TEST(FrameCropRegionComputerTest, HandlesEmptyDetections) {
+  const auto options = MakeKeyFrameCropOptions(kTargetWidth, kTargetHeight);
+  FrameCropRegionComputer computer(options);
+  KeyFrameInfo key_frame_info;
+  KeyFrameCropResult crop_result;
+  MP_EXPECT_OK(computer.ComputeFrameCropRegion(key_frame_info, &crop_result));
+  EXPECT_TRUE(crop_result.region_is_empty());
+}
+
+// Checks that ComputeFrameCropRegion covers required regions when their union
+// is within target size.
+TEST(FrameCropRegionComputerTest, CoversRequiredWithinTargetSize) {
+  const auto options = MakeKeyFrameCropOptions(kTargetWidth, kTargetHeight);
+  FrameCropRegionComputer computer(options);
+  KeyFrameInfo key_frame_info;
+  AddDetection(MakeRect(100, 100, 100, 200), true, &key_frame_info);
+  AddDetection(MakeRect(200, 400, 300, 500), true, &key_frame_info);
+  KeyFrameCropResult crop_result;
+  MP_EXPECT_OK(computer.ComputeFrameCropRegion(key_frame_info, &crop_result));
+  CheckRequiredRegionsAreCovered(key_frame_info, crop_result);
+  EXPECT_TRUE(CheckRectsEqual(MakeRect(100, 100, 400, 800),
+                              crop_result.required_region()));
+  EXPECT_TRUE(
+      CheckRectsEqual(crop_result.region(), crop_result.required_region()));
+  EXPECT_TRUE(crop_result.are_required_regions_covered_in_target_size());
+}
+
+// Checks that ComputeFrameCropRegion covers required regions when their union
+// exceeds target size.
+TEST(FrameCropRegionComputerTest, CoversRequiredExceedingTargetSize) {
+  const auto options = MakeKeyFrameCropOptions(kTargetWidth, kTargetHeight);
+  FrameCropRegionComputer computer(options);
+  KeyFrameInfo key_frame_info;
+  AddDetection(MakeRect(0, 0, 100, 500), true, &key_frame_info);
+  AddDetection(MakeRect(200, 400, 500, 500), true, &key_frame_info);
+  KeyFrameCropResult crop_result;
+  MP_EXPECT_OK(computer.ComputeFrameCropRegion(key_frame_info, &crop_result));
+  CheckRequiredRegionsAreCovered(key_frame_info, crop_result);
+  EXPECT_TRUE(CheckRectsEqual(MakeRect(0, 0, 700, 900), crop_result.region()));
+  EXPECT_TRUE(
+      CheckRectsEqual(crop_result.region(), crop_result.required_region()));
+  EXPECT_FALSE(crop_result.are_required_regions_covered_in_target_size());
+}
+
+// Checks that ComputeFrameCropRegion handles the case of only non-required
+// regions and the region fits in the target size.
+TEST(FrameCropRegionComputerTest,
+     HandlesOnlyNonRequiedRegionsInsideTargetSize) {
+  const auto options = MakeKeyFrameCropOptions(kTargetWidth, kTargetHeight);
+  FrameCropRegionComputer computer(options);
+  KeyFrameInfo key_frame_info;
+  AddDetection(MakeRect(300, 600, 100, 100), false, &key_frame_info);
+  KeyFrameCropResult crop_result;
+  MP_EXPECT_OK(computer.ComputeFrameCropRegion(key_frame_info, &crop_result));
+  EXPECT_TRUE(crop_result.required_region_is_empty());
+  EXPECT_FALSE(crop_result.region_is_empty());
+  EXPECT_TRUE(
+      CheckRectsEqual(key_frame_info.detections().detections(0).location(),
+                      crop_result.region()));
+}
+
+// Checks that ComputeFrameCropRegion handles the case of only non-required
+// regions and the region exceeds the target size.
+TEST(FrameCropRegionComputerTest,
+     HandlesOnlyNonRequiedRegionsExceedingTargetSize) {
+  const auto options = MakeKeyFrameCropOptions(kTargetWidth, kTargetHeight);
+  FrameCropRegionComputer computer(options);
+  KeyFrameInfo key_frame_info;
+  AddDetection(MakeRect(300, 600, 700, 100), false, &key_frame_info);
+  KeyFrameCropResult crop_result;
+  MP_EXPECT_OK(computer.ComputeFrameCropRegion(key_frame_info, &crop_result));
+  EXPECT_TRUE(crop_result.required_region_is_empty());
+  EXPECT_FALSE(crop_result.region_is_empty());
+  EXPECT_TRUE(
+      CheckRectsEqual(MakeRect(475, 600, 350, 100), crop_result.region()));
+  EXPECT_EQ(crop_result.fraction_non_required_covered(), 0.0);
+  EXPECT_TRUE(
+      CheckRectIsInside(crop_result.region(),
+                        key_frame_info.detections().detections(0).location()));
+}
+
+// Checks that ComputeFrameCropRegion covers non-required regions when their
+// union fits within target size.
+TEST(FrameCropRegionComputerTest, CoversNonRequiredInsideTargetSize) {
+  const auto options = MakeKeyFrameCropOptions(kTargetWidth, kTargetHeight);
+  FrameCropRegionComputer computer(options);
+  KeyFrameInfo key_frame_info;
+  AddDetection(MakeRect(0, 0, 100, 500), true, &key_frame_info);
+  AddDetection(MakeRect(300, 600, 100, 100), false, &key_frame_info);
+  KeyFrameCropResult crop_result;
+  MP_EXPECT_OK(computer.ComputeFrameCropRegion(key_frame_info, &crop_result));
+  CheckRequiredRegionsAreCovered(key_frame_info, crop_result);
+  EXPECT_TRUE(CheckRectsEqual(MakeRect(0, 0, 400, 700), crop_result.region()));
+  EXPECT_TRUE(crop_result.are_required_regions_covered_in_target_size());
+  EXPECT_EQ(crop_result.fraction_non_required_covered(), 1.0);
+  for (int i = 0; i < key_frame_info.detections().detections_size(); ++i) {
+    EXPECT_TRUE(
+        CheckRectIsInside(key_frame_info.detections().detections(i).location(),
+                          crop_result.region()));
+  }
+}
+
+// Checks that ComputeFrameCropRegion does not cover non-required regions that
+// are outside the target size.
+TEST(FrameCropRegionComputerTest, DoesNotCoverNonRequiredExceedingTargetSize) {
+  const auto options = MakeKeyFrameCropOptions(kTargetWidth, kTargetHeight);
+  FrameCropRegionComputer computer(options);
+  KeyFrameInfo key_frame_info;
+  AddDetection(MakeRect(0, 0, 500, 1000), true, &key_frame_info);
+  AddDetection(MakeRect(500, 0, 100, 100), false, &key_frame_info);
+  KeyFrameCropResult crop_result;
+  MP_EXPECT_OK(computer.ComputeFrameCropRegion(key_frame_info, &crop_result));
+  CheckRequiredRegionsAreCovered(key_frame_info, crop_result);
+  EXPECT_TRUE(CheckRectsEqual(MakeRect(0, 0, 500, 1000), crop_result.region()));
+  EXPECT_TRUE(crop_result.are_required_regions_covered_in_target_size());
+  EXPECT_EQ(crop_result.fraction_non_required_covered(), 0.0);
+  EXPECT_FALSE(
+      CheckRectIsInside(key_frame_info.detections().detections(1).location(),
+                        crop_result.region()));
+}
+
+// Checks that ComputeFrameCropRegion partially covers non-required regions that
+// can partially fit in the target size.
+TEST(FrameCropRegionComputerTest,
+     PartiallyCoversNonRequiredContainingTargetSize) {
+  const auto options = MakeKeyFrameCropOptions(kTargetWidth, kTargetHeight);
+  FrameCropRegionComputer computer(options);
+  KeyFrameInfo key_frame_info;
+  AddDetection(MakeRect(100, 0, 350, 1000), true, &key_frame_info);
+  AddDetection(MakeRect(0, 0, 650, 100), false, &key_frame_info);
+  KeyFrameCropResult crop_result;
+  MP_EXPECT_OK(computer.ComputeFrameCropRegion(key_frame_info, &crop_result));
+  CheckRequiredRegionsAreCovered(key_frame_info, crop_result);
+  EXPECT_TRUE(
+      CheckRectsEqual(MakeRect(100, 0, 387, 1000), crop_result.region()));
+  EXPECT_TRUE(crop_result.are_required_regions_covered_in_target_size());
+  EXPECT_EQ(crop_result.fraction_non_required_covered(), 0.0);
+  EXPECT_TRUE(
+      CheckRectsOverlap(key_frame_info.detections().detections(1).location(),
+                        crop_result.region()));
+}
+
+// Checks that ComputeFrameCropRegion covers non-required regions when the
+// required regions exceed target size.
+TEST(FrameCropRegionComputerTest,
+     CoversNonRequiredWhenRequiredExceedsTargetSize) {
+  const auto options = MakeKeyFrameCropOptions(kTargetWidth, kTargetHeight);
+  FrameCropRegionComputer computer(options);
+  KeyFrameInfo key_frame_info;
+  AddDetection(MakeRect(0, 0, 600, 1000), true, &key_frame_info);
+  AddDetection(MakeRect(450, 0, 100, 100), false, &key_frame_info);
+  KeyFrameCropResult crop_result;
+  MP_EXPECT_OK(computer.ComputeFrameCropRegion(key_frame_info, &crop_result));
+  CheckRequiredRegionsAreCovered(key_frame_info, crop_result);
+  EXPECT_TRUE(CheckRectsEqual(MakeRect(0, 0, 600, 1000), crop_result.region()));
+  EXPECT_FALSE(crop_result.are_required_regions_covered_in_target_size());
+  EXPECT_EQ(crop_result.fraction_non_required_covered(), 1.0);
+  for (int i = 0; i < key_frame_info.detections().detections_size(); ++i) {
+    EXPECT_TRUE(
+        CheckRectIsInside(key_frame_info.detections().detections(i).location(),
+                          crop_result.region()));
+  }
+}
+
+// Checks that ComputeFrameCropRegion does not extend the crop region when
+// the non-required region is too far.
+TEST(FrameCropRegionComputerTest,
+     DoesNotExtendRegionWhenNonRequiredRegionIsTooFar) {
+  const auto options = MakeKeyFrameCropOptions(kTargetWidth, kTargetHeight);
+  FrameCropRegionComputer computer(options);
+  KeyFrameInfo key_frame_info;
+  AddDetection(MakeRect(0, 0, 400, 400), true, &key_frame_info);
+  AddDetection(MakeRect(600, 0, 100, 100), false, &key_frame_info);
+  KeyFrameCropResult crop_result;
+  MP_EXPECT_OK(computer.ComputeFrameCropRegion(key_frame_info, &crop_result));
+  CheckRequiredRegionsAreCovered(key_frame_info, crop_result);
+  EXPECT_TRUE(CheckRectsEqual(MakeRect(0, 0, 400, 400), crop_result.region()));
+  EXPECT_TRUE(crop_result.are_required_regions_covered_in_target_size());
+  EXPECT_EQ(crop_result.fraction_non_required_covered(), 0.0);
+  EXPECT_FALSE(
+      CheckRectsOverlap(key_frame_info.detections().detections(1).location(),
+                        crop_result.region()));
+}
+
+// Checks that ComputeFrameCropRegion computes the score correctly when the
+// aggregation type is maximum.
+TEST(FrameCropRegionComputerTest, ComputesScoreWhenAggregationIsMaximum) {
+  auto options = MakeKeyFrameCropOptions(kTargetWidth, kTargetHeight);
+  options.set_score_aggregation_type(KeyFrameCropOptions::MAXIMUM);
+  FrameCropRegionComputer computer(options);
+  KeyFrameInfo key_frame_info;
+  AddDetection(MakeRect(0, 0, 400, 400), true, &key_frame_info, 0.1);
+  AddDetection(MakeRect(300, 300, 200, 500), true, &key_frame_info, 0.9);
+  KeyFrameCropResult crop_result;
+  MP_EXPECT_OK(computer.ComputeFrameCropRegion(key_frame_info, &crop_result));
+  EXPECT_FLOAT_EQ(crop_result.region_score(), 0.9f);
+}
+
+// Checks that ComputeFrameCropRegion computes the score correctly when the
+// aggregation type is sum required regions.
+TEST(FrameCropRegionComputerTest, ComputesScoreWhenAggregationIsSumRequired) {
+  auto options = MakeKeyFrameCropOptions(kTargetWidth, kTargetHeight);
+  options.set_score_aggregation_type(KeyFrameCropOptions::SUM_REQUIRED);
+  FrameCropRegionComputer computer(options);
+  KeyFrameInfo key_frame_info;
+  AddDetection(MakeRect(0, 0, 400, 400), true, &key_frame_info, 0.1);
+  AddDetection(MakeRect(300, 300, 200, 500), true, &key_frame_info, 0.9);
+  AddDetection(MakeRect(300, 300, 200, 500), false, &key_frame_info, 0.5);
+  KeyFrameCropResult crop_result;
+  MP_EXPECT_OK(computer.ComputeFrameCropRegion(key_frame_info, &crop_result));
+  EXPECT_FLOAT_EQ(crop_result.region_score(), 1.0f);
+}
+
+// Checks that ComputeFrameCropRegion computes the score correctly when the
+// aggregation type is sum all covered regions.
+TEST(FrameCropRegionComputerTest, ComputesScoreWhenAggregationIsSumAll) {
+  auto options = MakeKeyFrameCropOptions(kTargetWidth, kTargetHeight);
+  options.set_score_aggregation_type(KeyFrameCropOptions::SUM_ALL);
+  FrameCropRegionComputer computer(options);
+  KeyFrameInfo key_frame_info;
+  AddDetection(MakeRect(0, 0, 400, 400), true, &key_frame_info, 0.1);
+  AddDetection(MakeRect(300, 300, 200, 500), true, &key_frame_info, 0.9);
+  AddDetection(MakeRect(300, 300, 200, 500), false, &key_frame_info, 0.5);
+  KeyFrameCropResult crop_result;
+  MP_EXPECT_OK(computer.ComputeFrameCropRegion(key_frame_info, &crop_result));
+  EXPECT_FLOAT_EQ(crop_result.region_score(), 1.5f);
+}
+
+// Checks that ComputeFrameCropRegion computes the score correctly when the
+// aggregation type is constant.
+TEST(FrameCropRegionComputerTest, ComputesScoreWhenAggregationIsConstant) {
+  auto options = MakeKeyFrameCropOptions(kTargetWidth, kTargetHeight);
+  options.set_score_aggregation_type(KeyFrameCropOptions::CONSTANT);
+  FrameCropRegionComputer computer(options);
+  KeyFrameInfo key_frame_info;
+  AddDetection(MakeRect(0, 0, 400, 400), true, &key_frame_info, 0.1);
+  AddDetection(MakeRect(300, 300, 200, 500), true, &key_frame_info, 0.9);
+  AddDetection(MakeRect(300, 300, 200, 500), false, &key_frame_info, 0.5);
+  KeyFrameCropResult crop_result;
+  MP_EXPECT_OK(computer.ComputeFrameCropRegion(key_frame_info, &crop_result));
+  EXPECT_FLOAT_EQ(crop_result.region_score(), 1.0f);
+}
+}  // namespace autoflip
+}  // namespace mediapipe

mediapipe/examples/desktop/autoflip/quality/math_utils.h

@@ -0,0 +1,40 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_MATH_UTILS_H_
+#define MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_MATH_UTILS_H_
+
+class MathUtil {
+ public:
+  // Clamps value to the range [low, high].  Requires low <= high. Returns false
+  // if this check fails, otherwise returns true. Caller should first check the
+  // returned boolean.
+  template <typename T>  // T models LessThanComparable.
+  static bool Clamp(const T& low, const T& high, const T& value, T* result) {
+    // Prevents errors in ordering the arguments.
+    if (low > high) {
+      return false;
+    }
+    if (high < value) {
+      *result = high;
+    } else if (value < low) {
+      *result = low;
+    } else {
+      *result = value;
+    }
+    return true;
+  }
+};
+
+#endif  // MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_MATH_UTILS_H_

mediapipe/examples/desktop/autoflip/quality/padding_effect_generator.cc

@@ -0,0 +1,177 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/examples/desktop/autoflip/quality/padding_effect_generator.h"
+
+#include "mediapipe/framework/formats/image_frame.h"
+#include "mediapipe/framework/formats/image_frame_opencv.h"
+#include "mediapipe/framework/port/opencv_core_inc.h"
+#include "mediapipe/framework/port/opencv_imgproc_inc.h"
+#include "mediapipe/framework/port/ret_check.h"
+
+namespace mediapipe {
+namespace autoflip {
+
+PaddingEffectGenerator::PaddingEffectGenerator(const int input_width,
+                                               const int input_height,
+                                               const double target_aspect_ratio,
+                                               bool scale_to_multiple_of_two) {
+  target_aspect_ratio_ = target_aspect_ratio;
+  const double input_aspect_ratio =
+      static_cast<double>(input_width) / static_cast<double>(input_height);
+  input_width_ = input_width;
+  input_height_ = input_height;
+  is_vertical_padding_ = input_aspect_ratio > target_aspect_ratio;
+  output_width_ = is_vertical_padding_
+                      ? std::round(target_aspect_ratio * input_height)
+                      : input_width;
+  output_height_ = is_vertical_padding_
+                       ? input_height
+                       : std::round(input_width / target_aspect_ratio);
+  if (scale_to_multiple_of_two) {
+    output_width_ = output_width_ / 2 * 2;
+    output_height_ = output_height_ / 2 * 2;
+  }
+}
+
+::mediapipe::Status PaddingEffectGenerator::Process(
+    const ImageFrame& input_frame, const float background_contrast,
+    const int blur_cv_size, const float overlay_opacity,
+    ImageFrame* output_frame, const cv::Scalar* background_color_in_rgb) {
+  RET_CHECK_EQ(input_frame.Width(), input_width_);
+  RET_CHECK_EQ(input_frame.Height(), input_height_);
+  RET_CHECK(output_frame);
+
+  cv::Mat original_image = formats::MatView(&input_frame);
+  // This is the canvas that we are going to draw the padding effect on to.
+  cv::Mat canvas(output_height_, output_width_, original_image.type());
+
+  const int effective_input_width =
+      is_vertical_padding_ ? input_width_ : input_height_;
+  const int effective_input_height =
+      is_vertical_padding_ ? input_height_ : input_width_;
+  const int effective_output_width =
+      is_vertical_padding_ ? output_width_ : output_height_;
+  const int effective_output_height =
+      is_vertical_padding_ ? output_height_ : output_width_;
+
+  if (!is_vertical_padding_) {
+    original_image = original_image.t();
+    canvas = canvas.t();
+  }
+
+  const int foreground_height =
+      effective_input_height * effective_output_width / effective_input_width;
+  int x = -1, y = -1, width = -1, height = -1;
+
+  // The following steps does the padding operation, with several steps.
+  // #1, we prepare the background. If a solid background color is given, we use
+  //     it directly. Otherwise, we first crop a region of size "output_width_ *
+  //     output_height_" off of the original frame to become the background of
+  //     the final frame, and then we blur it and adjust contrast and opacity.
+  if (background_color_in_rgb != nullptr) {
+    canvas = *background_color_in_rgb;
+  } else {
+    // Copy the original image to the background.
+    x = 0.5 * (effective_input_width - effective_output_width);
+    y = 0;
+    width = effective_output_width;
+    height = effective_output_height;
+    cv::Rect crop_window_for_background(x, y, width, height);
+    original_image(crop_window_for_background).copyTo(canvas);
+
+    // Blur.
+    const int cv_size =
+        blur_cv_size % 2 == 1 ? blur_cv_size : (blur_cv_size + 1);
+    const cv::Size kernel(cv_size, cv_size);
+    // TODO: the larger the kernel size, the slower the blurring
+    // operation is. Consider running multiple sequential blurs with smaller
+    // sizes to simulate the effect of using a large size. This might be able to
+    // speed up the process.
+    x = 0;
+    width = effective_output_width;
+    const cv::Rect canvas_rect(0, 0, canvas.cols, canvas.rows);
+    // Blur the top region (above foreground).
+    y = 0;
+    height = (effective_output_height - foreground_height) / 2 + cv_size;
+    const cv::Rect top_blur_region =
+        cv::Rect(x, y, width, height) & canvas_rect;
+    if (top_blur_region.area() > 0) {
+      cv::Mat top_blurred = canvas(top_blur_region);
+      cv::GaussianBlur(top_blurred, top_blurred, kernel, 0, 0);
+    }
+    // Blur the bottom region (below foreground).
+    y = height + foreground_height - cv_size;
+    height = effective_output_height - y;
+    const cv::Rect bottom_blur_region =
+        cv::Rect(x, y, width, height) & canvas_rect;
+    if (bottom_blur_region.area() > 0) {
+      cv::Mat bottom_blurred = canvas(bottom_blur_region);
+      cv::GaussianBlur(bottom_blurred, bottom_blurred, kernel, 0, 0);
+    }
+
+    const float kEqualThreshold = 0.0001f;
+    // Background contrast adjustment.
+    if (std::abs(background_contrast - 1.0f) > kEqualThreshold) {
+      canvas *= background_contrast;
+    }
+
+    // Alpha blend a translucent black layer.
+    if (std::abs(overlay_opacity - 0.0f) > kEqualThreshold) {
+      cv::Mat overlay = cv::Mat::zeros(canvas.size(), canvas.type());
+      cv::addWeighted(overlay, overlay_opacity, canvas, 1 - overlay_opacity, 0,
+                      canvas);
+    }
+  }
+
+  // #2, we crop the entire region off of the original frame. This will become
+  //     the foreground in the final frame.
+  x = 0;
+  y = 0;
+  width = effective_input_width;
+  height = effective_input_height;
+
+  cv::Rect crop_window_for_foreground(x, y, width, height);
+
+  // #3, we specify a region of size computed as below in the final frame to
+  //     embed the foreground that we obtained in #2. The aspect ratio of
+  //     this region should be the same as the foreground, but with a
+  //     smaller size. Therefore, the height and width are derived using
+  //     the ratio of the sizes.
+  //     - embed size: output_width_ * height (to be computed)
+  //     - foreground: input_width * input_height
+  //
+  //     The location of this region is horizontally centralized in the
+  //     frame, and saturated in horizontal dimension.
+  x = 0;
+  y = (effective_output_height - foreground_height) / 2;
+  width = effective_output_width;
+  height = foreground_height;
+
+  cv::Rect region_to_embed_foreground(x, y, width, height);
+  cv::Mat dst = canvas(region_to_embed_foreground);
+  cv::resize(original_image(crop_window_for_foreground), dst, dst.size());
+
+  if (!is_vertical_padding_) {
+    canvas = canvas.t();
+  }
+
+  output_frame->CopyPixelData(input_frame.Format(), canvas.cols, canvas.rows,
+                              canvas.data,
+                              ImageFrame::kDefaultAlignmentBoundary);
+  return ::mediapipe::OkStatus();
+}
+
+}  // namespace autoflip
+}  // namespace mediapipe

mediapipe/examples/desktop/autoflip/quality/padding_effect_generator.h

@@ -0,0 +1,70 @@
+#ifndef MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_PADDING_EFFECT_GENERATOR_H_
+#define MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_PADDING_EFFECT_GENERATOR_H_
+
+#include "mediapipe/framework/formats/image_frame.h"
+#include "mediapipe/framework/port/opencv_core_inc.h"
+#include "mediapipe/framework/port/opencv_imgproc_inc.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+namespace autoflip {
+
+// Generates padding effects given input frames. Depending on where the padded
+// contents are added, there are two cases:
+// 1) Pad on the top and bottom of the input frame, aka vertical padding, i.e.
+//    input_aspect_ratio > target_aspect_ratio. In this case, output frames will
+//    have the same height as input frames, and the width will be adjusted to
+//    match the target aspect ratio.
+// 2) Pad on the left and right of the input frame, aka horizontal padding, i.e.
+//    input_aspect_ratio < target_aspect_ratio. In this case, output frames will
+//    have the same width as original frames, and the height will be adjusted to
+//    match the target aspect ratio.
+// If a background color is given, the background of the output frame will be
+// filled with this solid color; otherwise, it is a blurred version of the input
+// frame.
+//
+// Note: in both horizontal and vertical padding effects, the output frame size
+// will be at most as large as the input frame size, with one dimension the
+// same as the input (horizontal padding: width, vertical padding: height). If
+// you intented to have the output frame be larger, you could add a
+// ScaleImageCalculator as an upstream node before calling this calculator in
+// your MediaPipe graph (not as a downstream node, because visual details may
+// lose after appling the padding effect).
+class PaddingEffectGenerator {
+ public:
+  // Always outputs width and height that are divisible by 2 if
+  // scale_to_multiple_of_two is set to true.
+  PaddingEffectGenerator(const int input_width, const int input_height,
+                         const double target_aspect_ratio,
+                         bool scale_to_multiple_of_two = false);
+
+  // Apply the padding effect on the input frame.
+  // - blur_cv_size: The cv::Size() parameter used in creating blurry effects
+  //   for padding backgrounds.
+  // - background_contrast: Contrast adjustment for padding background. This
+  //   value should between 0 and 1, and the smaller the value, the darker the
+  //   background.
+  // - overlay_opacity: In addition to adjusting the contrast, a translucent
+  //   black layer will be alpha blended with the background. This value defines
+  //   the opacity of the black layer.
+  // - background_color_in_rgb: If not null, uses this solid color as background
+  //   instead of blurring the image, and does not adjust contrast or opacity.
+  ::mediapipe::Status Process(
+      const ImageFrame& input_frame, const float background_contrast,
+      const int blur_cv_size, const float overlay_opacity,
+      ImageFrame* output_frame,
+      const cv::Scalar* background_color_in_rgb = nullptr);
+
+ private:
+  double target_aspect_ratio_;
+  int input_width_ = -1;
+  int input_height_ = -1;
+  int output_width_ = -1;
+  int output_height_ = -1;
+  bool is_vertical_padding_;
+};
+
+}  // namespace autoflip
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_PADDING_EFFECT_GENERATOR_H_

mediapipe/examples/desktop/autoflip/quality/padding_effect_generator_test.cc

@@ -0,0 +1,187 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/examples/desktop/autoflip/quality/padding_effect_generator.h"
+
+#include "absl/strings/str_cat.h"
+#include "mediapipe/framework/deps/file_path.h"
+#include "mediapipe/framework/formats/image_frame.h"
+#include "mediapipe/framework/formats/image_frame_opencv.h"
+#include "mediapipe/framework/port/commandlineflags.h"
+#include "mediapipe/framework/port/file_helpers.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/opencv_imgcodecs_inc.h"
+#include "mediapipe/framework/port/opencv_imgproc_inc.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status_builder.h"
+#include "mediapipe/framework/port/status_matchers.h"
+
+DEFINE_string(input_image, "", "The path to an input image.");
+DEFINE_string(output_folder, "", "The folder to output test result images.");
+
+namespace mediapipe {
+namespace autoflip {
+namespace {
+
+// An 320x180 RGB test image.
+constexpr char kTestImage[] =
+    "mediapipe/examples/desktop/autoflip/quality/testdata/"
+    "google.jpg";
+constexpr char kResultImagePrefix[] =
+    "mediapipe/examples/desktop/autoflip/quality/testdata/"
+    "result_";
+
+const cv::Scalar kRed = cv::Scalar(255, 0, 0);
+
+void TestWithAspectRatio(const double aspect_ratio,
+                         const cv::Scalar* background_color_in_rgb = nullptr) {
+  std::string test_image;
+  const bool process_arbitrary_image = !FLAGS_input_image.empty();
+  if (!process_arbitrary_image) {
+    std::string test_image_path = mediapipe::file::JoinPath("./", kTestImage);
+    MP_ASSERT_OK(mediapipe::file::GetContents(test_image_path, &test_image));
+  } else {
+    MP_ASSERT_OK(mediapipe::file::GetContents(FLAGS_input_image, &test_image));
+  }
+
+  const std::vector<char> contents_vector(test_image.begin(), test_image.end());
+  cv::Mat decoded_mat =
+      cv::imdecode(contents_vector, -1 /* return the loaded image as-is */);
+
+  ImageFormat::Format image_format = ImageFormat::UNKNOWN;
+  cv::Mat output_mat;
+  switch (decoded_mat.channels()) {
+    case 1:
+      image_format = ImageFormat::GRAY8;
+      output_mat = decoded_mat;
+      break;
+    case 3:
+      image_format = ImageFormat::SRGB;
+      cv::cvtColor(decoded_mat, output_mat, cv::COLOR_BGR2RGB);
+      break;
+    case 4:
+      MP_ASSERT_OK(::mediapipe::UnimplementedErrorBuilder(MEDIAPIPE_LOC)
+                   << "4-channel image isn't supported yet");
+      break;
+    default:
+      MP_ASSERT_OK(::mediapipe::FailedPreconditionErrorBuilder(MEDIAPIPE_LOC)
+                   << "Unsupported number of channels: "
+                   << decoded_mat.channels());
+  }
+  std::unique_ptr<ImageFrame> test_frame = absl::make_unique<ImageFrame>(
+      image_format, decoded_mat.size().width, decoded_mat.size().height);
+  output_mat.copyTo(formats::MatView(test_frame.get()));
+
+  PaddingEffectGenerator generator(test_frame->Width(), test_frame->Height(),
+                                   aspect_ratio);
+  ImageFrame result_frame;
+  MP_ASSERT_OK(generator.Process(*test_frame, 0.3, 40, 0.0, &result_frame,
+                                 background_color_in_rgb));
+  cv::Mat original_mat = formats::MatView(&result_frame);
+  cv::Mat input_mat;
+  switch (original_mat.channels()) {
+    case 1:
+      input_mat = original_mat;
+      break;
+    case 3:
+      // OpenCV assumes the image to be BGR order. To use imencode(), do color
+      // conversion first.
+      cv::cvtColor(original_mat, input_mat, cv::COLOR_RGB2BGR);
+      break;
+    case 4:
+      MP_ASSERT_OK(::mediapipe::UnimplementedErrorBuilder(MEDIAPIPE_LOC)
+                   << "4-channel image isn't supported yet");
+      break;
+    default:
+      MP_ASSERT_OK(::mediapipe::FailedPreconditionErrorBuilder(MEDIAPIPE_LOC)
+                   << "Unsupported number of channels: "
+                   << original_mat.channels());
+  }
+
+  std::vector<int> parameters;
+  parameters.push_back(cv::IMWRITE_JPEG_QUALITY);
+  constexpr int kEncodingQuality = 75;
+  parameters.push_back(kEncodingQuality);
+
+  std::vector<uchar> encode_buffer;
+  // Note that imencode() will store the data in RGB order.
+  // Check its JpegEncoder::write() in "imgcodecs/src/grfmt_jpeg.cpp" for more
+  // info.
+  if (!cv::imencode(".jpg", input_mat, encode_buffer, parameters)) {
+    MP_ASSERT_OK(::mediapipe::InternalErrorBuilder(MEDIAPIPE_LOC)
+                 << "Fail to encode the image to be jpeg format.");
+  }
+
+  std::string output_string(absl::string_view(
+      reinterpret_cast<const char*>(&encode_buffer[0]), encode_buffer.size()));
+
+  if (!process_arbitrary_image) {
+    std::string result_string_path = mediapipe::file::JoinPath(
+        "./", absl::StrCat(kResultImagePrefix, aspect_ratio,
+                           background_color_in_rgb ? "_solid_background" : "",
+                           ".jpg"));
+    std::string result_image;
+    MP_ASSERT_OK(
+        mediapipe::file::GetContents(result_string_path, &result_image));
+    EXPECT_EQ(result_image, output_string);
+  } else {
+    std::string output_string_path = mediapipe::file::JoinPath(
+        FLAGS_output_folder,
+        absl::StrCat("result_", aspect_ratio,
+                     background_color_in_rgb ? "_solid_background" : "",
+                     ".jpg"));
+    MP_ASSERT_OK(
+        mediapipe::file::SetContents(output_string_path, output_string));
+  }
+}
+
+TEST(PaddingEffectGeneratorTest, Success) {
+  TestWithAspectRatio(0.3);
+  TestWithAspectRatio(0.6);
+  TestWithAspectRatio(1.0);
+  TestWithAspectRatio(1.6);
+  TestWithAspectRatio(2.5);
+  TestWithAspectRatio(3.4);
+}
+
+TEST(PaddingEffectGeneratorTest, SuccessWithBackgroundColor) {
+  TestWithAspectRatio(0.3, &kRed);
+  TestWithAspectRatio(0.6, &kRed);
+  TestWithAspectRatio(1.0, &kRed);
+  TestWithAspectRatio(1.6, &kRed);
+  TestWithAspectRatio(2.5, &kRed);
+  TestWithAspectRatio(3.4, &kRed);
+}
+
+TEST(PaddingEffectGeneratorTest, ScaleToMultipleOfTwo) {
+  int input_width = 30;
+  int input_height = 30;
+  double target_aspect_ratio = 0.5;
+  int expect_width = 14;
+  int expect_height = input_height;
+  auto test_frame = absl::make_unique<ImageFrame>(/*format=*/ImageFormat::SRGB,
+                                                  input_width, input_height);
+
+  PaddingEffectGenerator generator(test_frame->Width(), test_frame->Height(),
+                                   target_aspect_ratio,
+                                   /*scale_to_multiple_of_two=*/true);
+  ImageFrame result_frame;
+  MP_ASSERT_OK(generator.Process(*test_frame, 0.3, 40, 0.0, &result_frame));
+  EXPECT_EQ(result_frame.Width(), expect_width);
+  EXPECT_EQ(result_frame.Height(), expect_height);
+}
+}  // namespace
+}  // namespace autoflip
+}  // namespace mediapipe

mediapipe/examples/desktop/autoflip/quality/piecewise_linear_function.cc

@@ -0,0 +1,69 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/examples/desktop/autoflip/quality/piecewise_linear_function.h"
+
+#include <stddef.h>
+
+#include <algorithm>
+#include <limits>
+#include <vector>
+
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+namespace autoflip {
+
+void PiecewiseLinearFunction::AddPoint(double x, double y) {
+  if (!points_.empty()) {
+    CHECK_GE(x, points_.back().x)
+        << "Points must be provided in non-decreasing x order.";
+  }
+  points_.push_back(PiecewiseLinearFunction::Point(x, y));
+}
+
+std::vector<PiecewiseLinearFunction::Point>::const_iterator
+PiecewiseLinearFunction::GetIntervalIterator(double input) const {
+  PiecewiseLinearFunction::Point input_point(input, 0);
+  std::vector<PiecewiseLinearFunction::Point>::const_iterator iter =
+      std::lower_bound(points_.begin(), points_.end(), input_point,
+                       PointCompare());
+  return iter;
+}
+
+double PiecewiseLinearFunction::Interpolate(
+    const PiecewiseLinearFunction::Point& p1,
+    const PiecewiseLinearFunction::Point& p2, double input) const {
+  CHECK_LT(p1.x, input);
+  CHECK_GE(p2.x, input);
+
+  return p2.y - (p2.x - input) / (p2.x - p1.x) * (p2.y - p1.y);
+}
+
+double PiecewiseLinearFunction::Evaluate(double const input) const {
+  std::vector<PiecewiseLinearFunction::Point>::const_iterator i =
+      GetIntervalIterator(input);
+  if (i == points_.begin()) {
+    return points_.front().y;
+  }
+  if (i == points_.end()) {
+    return points_.back().y;
+  }
+
+  std::vector<PiecewiseLinearFunction::Point>::const_iterator prev = i - 1;
+  return Interpolate(*prev, *i, input);
+}
+
+}  // namespace autoflip
+}  // namespace mediapipe

mediapipe/examples/desktop/autoflip/quality/piecewise_linear_function.h

@@ -0,0 +1,82 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_PIECEWISE_LINEAR_FUNCTION_H_
+#define MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_PIECEWISE_LINEAR_FUNCTION_H_
+
+#include <vector>
+
+namespace mediapipe {
+namespace autoflip {
+
+// Implementation of piecewise linear functions. The function is specified as a
+// series of points (x1,y1), (x2,y2),..., (xn,yn). It can be constructed
+// programmatically by repeatedly calling the AddPoint(x, y) method.
+class PiecewiseLinearFunction {
+ public:
+  PiecewiseLinearFunction() {}
+
+  // Evaluate the function at the specified input. The output
+  // saturates at the values of the first and last interpolation
+  // points.
+  // f(x) = y1 for x <= x1   // Saturate at the lowest value
+  // f(x) = yn for x >  xn   // Saturate at the highest value
+  // f(x) = (x-xj)/(xk-xj)*(yk-yj) + yk for xj < x <= xk and k = j+1
+  double Evaluate(double input) const;
+
+  // Adds the given point to the function.  Points must be added in
+  // non-decreasing x order.  Because the points are given in sorted
+  // order, this function can be used to construct discontinuous
+  // functions.  For example, if one defines
+  //    f.AddPoint(-1.0, 0.0)
+  //    f.AddPoint( 0.0, 0.0)
+  //    f.AddPoint( 0.0, 1.0)
+  //    f.AddPoint( 1.0, 1.0)
+  // the result function f is discontinuous at 0.0.  By convention,
+  // this function will return f.Evaluate(0.0) = 0.0, and
+  // f.Evaluate(1e-12) = 1.0.  This convention corresponds to the
+  // natural behavior of GetIntervalIterator().
+  void AddPoint(double x, double y);
+
+ private:
+  struct Point {
+    double x;
+    double y;
+    Point(double X, double Y) : x(X), y(Y) {}
+  };
+
+  // A functor for use with stl algorithms like sort() and lower_bound() that
+  // sorts by the point's x value.
+  class PointCompare {
+   public:
+    bool operator()(const Point& p1, const Point& p2) const {
+      return p1.x < p2.x;
+    }
+  };
+
+  // Returns the iterator, i, closest to points_.begin() such that
+  // input <= i->x or it returns points_.end() if input > all x values
+  // in points_.
+  std::vector<Point>::const_iterator GetIntervalIterator(double input) const;
+
+  // Given two points p1 and p2 such that p1.x < input and p2.x >= input this
+  // returns the linear interpolation of the y value.
+  double Interpolate(const Point& p1, const Point& p2, double input) const;
+
+  std::vector<Point> points_;
+};
+
+}  // namespace autoflip
+}  // namespace mediapipe
+#endif  // MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_PIECEWISE_LINEAR_FUNCTION_H_

mediapipe/examples/desktop/autoflip/quality/piecewise_linear_function_test.cc

@@ -0,0 +1,73 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/examples/desktop/autoflip/quality/piecewise_linear_function.h"
+
+#include <stddef.h>
+
+#include <memory>
+
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace {
+
+using mediapipe::autoflip::PiecewiseLinearFunction;
+
+// It should be OK to pass a spec that's out of order as it gets sorted.
+TEST(PiecewiseLinearFunctionTest, ReordersSpec) {
+  PiecewiseLinearFunction f;
+  // This defines the line y = x between 0 and 5
+  f.AddPoint(0, 0);
+  f.AddPoint(1, 1);
+  f.AddPoint(2, 2);
+  f.AddPoint(3, 3);
+  f.AddPoint(5, 5);
+
+  // Should be 0 as -1 is less than the smallest x value in the spec so it
+  // should saturate.
+  ASSERT_EQ(0, f.Evaluate(-1));
+
+  // These shoud all be on the line y = x
+  ASSERT_EQ(0, f.Evaluate(0));
+  ASSERT_EQ(0.5, f.Evaluate(0.5));
+  ASSERT_EQ(4.5, f.Evaluate(4.5));
+  ASSERT_EQ(5, f.Evaluate(5));
+
+  // Saturating on the high end.
+  ASSERT_EQ(5, f.Evaluate(6));
+}
+
+TEST(PiecewiseLinearFunctionTest, TestAddPoints) {
+  PiecewiseLinearFunction function;
+  function.AddPoint(0.0, 0.0);
+  function.AddPoint(1.0, 1.0);
+  EXPECT_DOUBLE_EQ(0.0, function.Evaluate(-1.0));
+  EXPECT_DOUBLE_EQ(0.0, function.Evaluate(0.0));
+  EXPECT_DOUBLE_EQ(0.25, function.Evaluate(0.25));
+}
+
+TEST(PiecewiseLinearFunctionTest, AddPointsDiscontinuous) {
+  PiecewiseLinearFunction function;
+  function.AddPoint(-1.0, 0.0);
+  function.AddPoint(0.0, 0.0);
+  function.AddPoint(0.0, 1.0);
+  function.AddPoint(1.0, 1.0);
+  EXPECT_DOUBLE_EQ(0.0, function.Evaluate(-1.0));
+  EXPECT_DOUBLE_EQ(0.0, function.Evaluate(0.0));
+  EXPECT_DOUBLE_EQ(1.0, function.Evaluate(1e-12));
+  EXPECT_DOUBLE_EQ(1.0, function.Evaluate(3.14));
+}
+
+}  // namespace

mediapipe/examples/desktop/autoflip/quality/polynomial_regression_path_solver.cc

@@ -0,0 +1,160 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/examples/desktop/autoflip/quality/polynomial_regression_path_solver.h"
+
+#include "ceres/autodiff_cost_function.h"
+#include "ceres/cost_function.h"
+#include "ceres/loss_function.h"
+#include "ceres/solver.h"
+#include "mediapipe/examples/desktop/autoflip/quality/focus_point.pb.h"
+#include "mediapipe/framework/port/opencv_core_inc.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+namespace autoflip {
+
+using ceres::AutoDiffCostFunction;
+using ceres::CauchyLoss;
+using ceres::CostFunction;
+using ceres::Problem;
+using ceres::Solve;
+using ceres::Solver;
+
+namespace {
+
+// A residual operator that computes the error using polynomial fitting.
+struct PolynomialResidual {
+  PolynomialResidual(double in, double out) : in_(in), out_(out) {}
+
+  template <typename T>
+  bool operator()(const T* const a, const T* const b, const T* const c,
+                  const T* const d, const T* const k, T* residual) const {
+    residual[0] = out_ - a[0] * in_ - b[0] * in_ * in_ -
+                  c[0] * in_ * in_ * in_ - d[0] * in_ * in_ * in_ * in_ - k[0];
+    return true;
+  }
+
+ private:
+  const double in_;
+  const double out_;
+};
+
+float ComputeDelta(const float in, const int original_dimension,
+                   const int output_dimension, const double a, const double b,
+                   const double c, const double d, const double k) {
+  float out =
+      a * in + b * in * in + c * in * in * in + d * in * in * in * in + k;
+  float delta = (out - 0.5) * 2 * output_dimension;
+  const float max_delta = (original_dimension - output_dimension) / 2.0f;
+
+  // Make sure delta doesn't move the camera off the frame boundary.
+  if (delta > max_delta) {
+    delta = max_delta;
+  } else if (delta < -max_delta) {
+    delta = -max_delta;
+  }
+  return delta;
+}
+}  // namespace
+
+void PolynomialRegressionPathSolver::AddCostFunctionToProblem(
+    const double in, const double out, Problem* problem, double* a, double* b,
+    double* c, double* d, double* k) {
+  // Creating a cost function, with 1D residual and 5 1D parameter blocks. This
+  // is what the "1, 1, 1, 1, 1, 1" std::string below means.
+  CostFunction* cost_function =
+      new AutoDiffCostFunction<PolynomialResidual, 1, 1, 1, 1, 1, 1>(
+          new PolynomialResidual(in, out));
+  VLOG(1) << "------- adding " << in << ": " << out;
+  problem->AddResidualBlock(cost_function, new CauchyLoss(0.5), a, b, c, d, k);
+}
+
+::mediapipe::Status PolynomialRegressionPathSolver::ComputeCameraPath(
+    const std::vector<FocusPointFrame>& focus_point_frames,
+    const std::vector<FocusPointFrame>& prior_focus_point_frames,
+    const int original_width, const int original_height, const int output_width,
+    const int output_height, std::vector<cv::Mat>* all_xforms) {
+  RET_CHECK_GE(original_width, output_width);
+  RET_CHECK_GE(original_height, output_height);
+  const bool should_solve_x_problem = original_width != output_width;
+  const bool should_solve_y_problem = original_height != output_height;
+  RET_CHECK_GT(focus_point_frames.size() + prior_focus_point_frames.size(), 0);
+  Problem problem_x, problem_y;
+  for (int i = 0; i < prior_focus_point_frames.size(); ++i) {
+    const auto& spf = prior_focus_point_frames[i];
+    for (const auto& sp : spf.point()) {
+      const double center_x = sp.norm_point_x();
+      const double center_y = sp.norm_point_y();
+      const auto t = i;
+      if (should_solve_x_problem) {
+        AddCostFunctionToProblem(t, center_x, &problem_x, &xa_, &xb_, &xc_,
+                                 &xd_, &xk_);
+      }
+      if (should_solve_y_problem) {
+        AddCostFunctionToProblem(t, center_y, &problem_y, &ya_, &yb_, &yc_,
+                                 &yd_, &yk_);
+      }
+    }
+  }
+  for (int i = 0; i < focus_point_frames.size(); ++i) {
+    const auto& spf = focus_point_frames[i];
+    for (const auto& sp : spf.point()) {
+      const double center_x = sp.norm_point_x();
+      const double center_y = sp.norm_point_y();
+      const auto t = i + prior_focus_point_frames.size();
+      if (should_solve_x_problem) {
+        AddCostFunctionToProblem(t, center_x, &problem_x, &xa_, &xb_, &xc_,
+                                 &xd_, &xk_);
+      }
+      if (should_solve_y_problem) {
+        AddCostFunctionToProblem(t, center_y, &problem_y, &ya_, &yb_, &yc_,
+                                 &yd_, &yk_);
+      }
+    }
+  }
+
+  Solver::Options options;
+  options.linear_solver_type = ceres::DENSE_QR;
+
+  Solver::Summary summary;
+  Solve(options, &problem_x, &summary);
+  all_xforms->clear();
+  for (int i = 0;
+       i < focus_point_frames.size() + prior_focus_point_frames.size(); i++) {
+    // Code below assigns values into an affine model, defined as:
+    //  [1 0 dx]
+    //  [0 1 dy]
+    // When the camera moves along x axis, we assign delta to dx; otherwise we
+    // assign delta to dy.
+    cv::Mat transform = cv::Mat::eye(2, 3, CV_32FC1);
+    const float in = static_cast<float>(i);
+    if (should_solve_x_problem) {
+      const float delta = ComputeDelta(in, original_width, output_width, xa_,
+                                       xb_, xc_, xd_, xk_);
+      transform.at<float>(0, 2) = delta;
+    }
+    if (should_solve_y_problem) {
+      const float delta = ComputeDelta(in, original_height, output_height, ya_,
+                                       yb_, yc_, yd_, yk_);
+      transform.at<float>(1, 2) = delta;
+    }
+    all_xforms->push_back(transform);
+  }
+  return mediapipe::OkStatus();
+}
+
+}  // namespace autoflip
+}  // namespace mediapipe

mediapipe/examples/desktop/autoflip/quality/polynomial_regression_path_solver.h

@@ -0,0 +1,69 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_POLYNOMIAL_REGRESSION_PATH_SOLVER_H_
+#define MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_POLYNOMIAL_REGRESSION_PATH_SOLVER_H_
+
+#include "ceres/problem.h"
+#include "mediapipe/examples/desktop/autoflip/quality/focus_point.pb.h"
+#include "mediapipe/framework/port/opencv_core_inc.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+namespace autoflip {
+
+class PolynomialRegressionPathSolver {
+ public:
+  PolynomialRegressionPathSolver()
+      : xa_(0.0),
+        xb_(0.0),
+        xc_(0.0),
+        xd_(0.0),
+        xk_(0.0),
+        ya_(0.0),
+        yb_(0.0),
+        yc_(0.0),
+        yd_(0.0),
+        yk_(0.0) {}
+
+  // Given a series of focus points on frames, uses polynomial regression to
+  // compute a best guess of a 1D camera movement trajectory along x-axis and
+  // y-axis, such that focus points can be preserved as much as possible. The
+  // returned |all_xforms| hold the camera location at each timestamp
+  // corresponding to each input frame.
+  ::mediapipe::Status ComputeCameraPath(
+      const std::vector<FocusPointFrame>& focus_point_frames,
+      const std::vector<FocusPointFrame>& prior_focus_point_frames,
+      const int original_width, const int original_height,
+      const int output_width, const int output_height,
+      std::vector<cv::Mat>* all_xforms);
+
+ private:
+  // Adds a new cost function, constructed using |in| and |out|, into |problem|.
+  void AddCostFunctionToProblem(const double in, const double out,
+                                ceres::Problem* problem, double* a, double* b,
+                                double* c, double* d, double* k);
+
+  // The current implementation fixes the polynomial order at 4, i.e. the
+  // equation to estimate is: out = a * in + b * in^2 + c * in^3 + d * in^4 + k.
+  // The two sets of parameters below are for estimating trajectories along
+  // x-axis and y-axis, respectively.
+  double xa_, xb_, xc_, xd_, xk_;
+  double ya_, yb_, yc_, yd_, yk_;
+};
+
+}  // namespace autoflip
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_POLYNOMIAL_REGRESSION_PATH_SOLVER_H_

mediapipe/examples/desktop/autoflip/quality/polynomial_regression_path_solver_test.cc

@@ -0,0 +1,246 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/examples/desktop/autoflip/quality/polynomial_regression_path_solver.h"
+
+#include "mediapipe/examples/desktop/autoflip/quality/focus_point.pb.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/opencv_core_inc.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/port/status_matchers.h"
+
+namespace mediapipe {
+namespace autoflip {
+namespace {
+
+// A series of focus point locations from a real video.
+constexpr int kNumObservations = 291;
+constexpr double data[] = {
+    1,   0.4072740648, 2,   0.406096287,  3,   0.4049185093, 4,   0.4037407361,
+    5,   0.402562963,  6,   0.4013851806, 7,   0.4002074074, 8,   0.399029625,
+    9,   0.3978518519, 10,  0.3966740741, 11,  0.3954963056, 12,  0.3943185231,
+    13,  0.39314075,   14,  0.3919629676, 15,  0.3907851944, 16,  0.3896074167,
+    17,  0.3884296435, 18,  0.3872518611, 19,  0.386074088,  20,  0.3848963194,
+    21,  0.3837185417, 22,  0.3825407685, 23,  0.3813629861, 24,  0.3801852037,
+    25,  0.3790074398, 26,  0.3778296574, 27,  0.376651875,  28,  0.3754741111,
+    29,  0.3742963287, 30,  0.3731185509, 31,  0.3719407685, 32,  0.3707629861,
+    33,  0.3695852222, 34,  0.3684074398, 35,  0.3672296759, 36,  0.3661564352,
+    37,  0.3651877315, 38,  0.3643235509, 39,  0.3635639213, 40,  0.3629088241,
+    41,  0.36235825,   42,  0.3619122222, 43,  0.3615707269, 44,  0.3613337639,
+    45,  0.3612013519, 46,  0.3611734583, 47,  0.3612500926, 48,  0.3614312778,
+    49,  0.361717,     50,  0.3621072546, 51,  0.362602037,  52,  0.3632013519,
+    53,  0.3639052083, 54,  0.3647136111, 55,  0.3656265417, 56,  0.3666440046,
+    57,  0.3677659907, 58,  0.3689925139, 59,  0.3703235926, 60,  0.3716546528,
+    61,  0.3729857269, 62,  0.374316787,  63,  0.3756478611, 64,  0.3769789259,
+    65,  0.37831,      66,  0.3796410648, 67,  0.3809721296, 68,  0.3823031944,
+    69,  0.3836342685, 70,  0.384965338,  71,  0.3862963981, 72,  0.3876274676,
+    73,  0.388958537,  74,  0.3902290324, 75,  0.391438963,  76,  0.3925883241,
+    77,  0.3936771204, 78,  0.3947053426, 79,  0.3956729954, 80,  0.396580088,
+    81,  0.3974265972, 82,  0.3982125509, 83,  0.3989379259, 84,  0.3996027407,
+    85,  0.4002069815, 86,  0.400750662,  87,  0.4012337639, 88,  0.4016562963,
+    89,  0.4020182731, 90,  0.4023196667, 91,  0.4025604954, 92,  0.4027407593,
+    93,  0.4028604491, 94,  0.4029195787, 95,  0.4029181296, 96,  0.4028561204,
+    97,  0.4027407593, 98,  0.4026254028, 99,  0.4025100417, 100, 0.4023946852,
+    101, 0.4022793241, 102, 0.402163963,  103, 0.4020486065, 104, 0.4019332454,
+    105, 0.4018178889, 106, 0.4017025278, 107, 0.4015871759, 108, 0.4014718102,
+    109, 0.4013564491, 110, 0.4012410972, 111, 0.4011257315, 112, 0.4010103704,
+    113, 0.4008950185, 114, 0.400779662,  115, 0.4008743935, 116, 0.4011792083,
+    117, 0.4016941157, 118, 0.4024191111, 119, 0.4033541944, 120, 0.4044993704,
+    121, 0.4058546343, 122, 0.4074199815, 123, 0.4091954259, 124, 0.4111809537,
+    125, 0.4133765741, 126, 0.415782287,  127, 0.4183980787, 128, 0.4212239676,
+    129, 0.4242599352, 130, 0.427506,     131, 0.4309621528, 132, 0.4346283935,
+    133, 0.4385047222, 134, 0.4425911407, 135, 0.4468876481, 136, 0.4513942454,
+    137, 0.4561109306, 138, 0.4608276204, 139, 0.4655443056, 140, 0.4702609861,
+    141, 0.4749776736, 142, 0.4796943574, 143, 0.4844110435, 144, 0.4891277287,
+    145, 0.4938444148, 146, 0.4985611,    147, 0.5032777852, 148, 0.5079944708,
+    149, 0.512711156,  150, 0.5174278403, 151, 0.5221445259, 152, 0.526861212,
+    153, 0.5315778981, 154, 0.5362945833, 155, 0.5410112662, 156, 0.5457279537,
+    157, 0.5504446435, 158, 0.5551613241, 159, 0.5598780093, 160, 0.5643503102,
+    161, 0.5685782454, 162, 0.572561787,  163, 0.5763009583, 164, 0.5797957546,
+    165, 0.583046162,  166, 0.5860521944, 167, 0.5888138611, 168, 0.5913311296,
+    169, 0.5936040278, 170, 0.5956325463, 171, 0.5974166806, 172, 0.5989564491,
+    173, 0.6002518287, 174, 0.6013028241, 175, 0.6021094491, 176, 0.6026716944,
+    177, 0.6029895556, 178, 0.6030630556, 179, 0.602892162,  180, 0.6024768889,
+    181, 0.6018172361, 182, 0.6009132083, 183, 0.5997648009, 184, 0.5983720139,
+    185, 0.5967348519, 186, 0.595057662,  187, 0.5933804769, 188, 0.5917032963,
+    189, 0.5900261065, 190, 0.588348912,  191, 0.5866717315, 192, 0.5849945417,
+    193, 0.5833173519, 194, 0.5816401713, 195, 0.5799629861, 196, 0.578285787,
+    197, 0.5766086111, 198, 0.5749314213, 199, 0.5732542315, 200, 0.5715770509,
+    201, 0.5698998611, 202, 0.5682226667, 203, 0.5665454861, 204, 0.5648682963,
+    205, 0.5631911157, 206, 0.5615139213, 207, 0.559989287,  208, 0.5586172083,
+    209, 0.5573976713, 210, 0.5563306944, 211, 0.5554162662, 212, 0.5546543889,
+    213, 0.5540450648, 214, 0.5535882917, 215, 0.5532840694, 216, 0.5531324028,
+    217, 0.5531332824, 218, 0.5532867176, 219, 0.5535927083, 220, 0.5540512454,
+    221, 0.5546623333, 222, 0.5554259769, 223, 0.5563421667, 224, 0.5574109148,
+    225, 0.5586322083, 226, 0.5600060556, 227, 0.5615324583, 228, 0.563211412,
+    229, 0.565042912,  230, 0.5670269722, 231, 0.5691635833, 232, 0.5714527315,
+    233, 0.5738944491, 234, 0.576488713,  235, 0.5792355231, 236, 0.581982338,
+    237, 0.5847291528, 238, 0.5874759676, 239, 0.5902227824, 240, 0.5929695972,
+    241, 0.5957164074, 242, 0.5984632269, 243, 0.601210037,  244, 0.6039568565,
+    245, 0.6067036667, 246, 0.6094504815, 247, 0.6121973056, 248, 0.6149441111,
+    249, 0.6176909352, 250, 0.62043775,   251, 0.6231845556, 252, 0.6258408148,
+    253, 0.628406537,  254, 0.6308817269, 255, 0.6332663426, 256, 0.6355604167,
+    257, 0.6377639352, 258, 0.6398769259, 259, 0.6418993519, 260, 0.6438312407,
+    261, 0.6456725787, 262, 0.6474233704, 263, 0.6490836111, 264, 0.650653287,
+    265, 0.6521324444, 266, 0.653521037,  267, 0.6548190926, 268, 0.656026588,
+    269, 0.6571435417, 270, 0.6581699537, 271, 0.6591058102, 272, 0.6599511204,
+    273, 0.6607058796, 274, 0.6613700926, 275, 0.6620343056, 276, 0.6626985185,
+    277, 0.6633627454, 278, 0.6640269537, 279, 0.6646911759, 280, 0.6653553843,
+    281, 0.6660195972, 282, 0.6666838056, 283, 0.6673480278, 284, 0.6680122361,
+    285, 0.668676463,  286, 0.6693406713, 287, 0.6700048935, 288, 0.6706691019,
+    289, 0.6713333333, 290, 0.671997537,  291, 0.6726617454,
+};
+
+constexpr double prediction[] = {
+    18.885935,   16.560495,  14.316487,  12.15229,   10.066307,   8.056951,
+    6.1226487,   4.2618394,  2.4729967,  0.7545829,  -0.894928,   -2.47702,
+    -3.9931893,  -5.4449024, -6.833619,  -8.160782,  -9.427822,   -10.63615,
+    -11.78717,   -12.882269, -13.922811, -14.910168, -15.845669,  -16.730648,
+    -17.566418,  -18.354284, -19.095533, -19.791435, -20.443249,  -21.052212,
+    -21.619564,  -22.146511, -22.634256, -23.08399,  -23.496883,  -23.874086,
+    -24.216753,  -24.526012, -24.80297,  -25.048738, -25.264395,  -25.451015,
+    -25.609661,  -25.74137,  -25.84718,  -25.928093, -25.985123,  -26.01925,
+    -26.031458,  -26.022684, -25.993896, -25.946003, -25.879932,  -25.796581,
+    -25.696838,  -25.581581, -25.451654, -25.307919, -25.151188,  -24.982292,
+    -24.802023,  -24.611176, -24.410517, -24.200804, -23.98278,   -23.757189,
+    -23.52473,   -23.286121, -23.042034, -22.79315,  -22.540123,  -22.283602,
+    -22.024214,  -21.762579, -21.499294, -21.234953, -20.970118,  -20.70536,
+    -20.441223,  -20.178228, -19.916899, -19.65773,  -19.401217,  -19.147831,
+    -18.898027,  -18.65226,  -18.410952, -18.174517, -17.943365,  -17.717875,
+    -17.498428,  -17.285383, -17.079079, -16.879856, -16.688019,  -16.503883,
+    -16.327726,  -16.15982,  -16.000439, -15.849811, -15.7081785, -15.575754,
+    -15.452737,  -15.339321, -15.235674, -15.141964, -15.058327,  -14.9848995,
+    -14.9218025, -14.869129, -14.826971, -14.795404, -14.774489,  -14.764267,
+    -14.764768,  -14.776015, -14.798016, -14.830744, -14.874178,  -14.9282875,
+    -14.993011,  -15.068281, -15.15401,  -15.250105, -15.356457,  -15.472937,
+    -15.599405,  -15.73571,  -15.881681, -16.03713,  -16.201872,  -16.37569,
+    -16.558355,  -16.749626, -16.94926,  -17.156982, -17.372507,  -17.595535,
+    -17.825771,  -18.062872, -18.306505, -18.55632,  -18.811947,  -19.072998,
+    -19.339085,  -19.609785, -19.884687, -20.16334,  -20.4453,    -20.730091,
+    -21.017235,  -21.306234, -21.59658,  -21.887743, -22.179192,  -22.470362,
+    -22.760695,  -23.049604, -23.336494, -23.620754, -23.901754,  -24.17887,
+    -24.451435,  -24.718779, -24.980236, -25.235092, -25.482649,  -25.722181,
+    -25.952942,  -26.174181, -26.38514,  -26.585024, -26.77304,   -26.948387,
+    -27.110231,  -27.257734, -27.39005,  -27.506304, -27.605618,  -27.68709,
+    -27.749819,  -27.792877, -27.81533,  -27.816212, -27.794569,  -27.749413,
+    -27.679752,  -27.584576, -27.462858, -27.313555, -27.135622,  -26.927996,
+    -26.689583,  -26.419294, -26.11602,  -25.778639, -25.40601,   -24.996979,
+    -24.550379,  -24.06503,
+};
+
+void GenerateDataPoints(
+    const int focus_point_frames_length,
+    const int prior_focus_point_frames_length,
+    std::vector<FocusPointFrame>* focus_point_frames,
+    std::vector<FocusPointFrame>* prior_focus_point_frames) {
+  CHECK(focus_point_frames_length + prior_focus_point_frames_length <=
+        kNumObservations);
+  for (int i = 0; i < prior_focus_point_frames_length; i++) {
+    FocusPoint sp;
+    sp.set_norm_point_x(data[i]);
+    FocusPointFrame spf;
+    *spf.add_point() = sp;
+    prior_focus_point_frames->push_back(spf);
+  }
+  for (int i = 0; i < focus_point_frames_length; i++) {
+    FocusPoint sp;
+    sp.set_norm_point_x(data[i]);
+    FocusPointFrame spf;
+    *spf.add_point() = sp;
+    focus_point_frames->push_back(spf);
+  }
+}
+
+TEST(PolynomialRegressionPathSolverTest, Success) {
+  PolynomialRegressionPathSolver solver;
+  std::vector<FocusPointFrame> focus_point_frames;
+  std::vector<FocusPointFrame> prior_focus_point_frames;
+  std::vector<cv::Mat> all_xforms;
+  GenerateDataPoints(/* focus_point_frames_length = */ 100,
+                     /* prior_focus_point_frames_length = */ 100,
+                     &focus_point_frames, &prior_focus_point_frames);
+  constexpr int kFrameWidth = 200;
+  constexpr int kFrameHeight = 300;
+  constexpr int kCropWidth = 100;
+  constexpr int kCropHeight = 300;
+  MP_ASSERT_OK(solver.ComputeCameraPath(
+      focus_point_frames, prior_focus_point_frames, kFrameWidth, kFrameHeight,
+      kCropWidth, kCropHeight, &all_xforms));
+  ASSERT_EQ(all_xforms.size(), 200);
+  for (int i = 0; i < all_xforms.size(); i++) {
+    cv::Mat mat = all_xforms[i];
+    EXPECT_FLOAT_EQ(mat.at<float>(0, 2), prediction[i]);
+  }
+}
+
+TEST(PolynomialRegressionPathSolverTest, FewFramesShouldWork) {
+  PolynomialRegressionPathSolver solver;
+  std::vector<FocusPointFrame> focus_point_frames;
+  std::vector<FocusPointFrame> prior_focus_point_frames;
+  std::vector<cv::Mat> all_xforms;
+  GenerateDataPoints(/* focus_point_frames_length = */ 1,
+                     /* prior_focus_point_frames_length = */ 1,
+                     &focus_point_frames, &prior_focus_point_frames);
+  constexpr int kFrameWidth = 200;
+  constexpr int kFrameHeight = 300;
+  constexpr int kCropWidth = 100;
+  constexpr int kCropHeight = 300;
+  MP_ASSERT_OK(solver.ComputeCameraPath(
+      focus_point_frames, prior_focus_point_frames, kFrameWidth, kFrameHeight,
+      kCropWidth, kCropHeight, &all_xforms));
+  ASSERT_EQ(all_xforms.size(), 2);
+}
+
+TEST(PolynomialRegressionPathSolverTest, OneCurrentFrameShouldWork) {
+  PolynomialRegressionPathSolver solver;
+  std::vector<FocusPointFrame> focus_point_frames;
+  std::vector<FocusPointFrame> prior_focus_point_frames;
+  std::vector<cv::Mat> all_xforms;
+  GenerateDataPoints(/* focus_point_frames_length = */ 1,
+                     /* prior_focus_point_frames_length = */ 0,
+                     &focus_point_frames, &prior_focus_point_frames);
+  constexpr int kFrameWidth = 200;
+  constexpr int kFrameHeight = 300;
+  constexpr int kCropWidth = 100;
+  constexpr int kCropHeight = 300;
+  MP_ASSERT_OK(solver.ComputeCameraPath(
+      focus_point_frames, prior_focus_point_frames, kFrameWidth, kFrameHeight,
+      kCropWidth, kCropHeight, &all_xforms));
+  ASSERT_EQ(all_xforms.size(), 1);
+}
+
+TEST(PolynomialRegressionPathSolverTest, ZeroFrameShouldFail) {
+  PolynomialRegressionPathSolver solver;
+  std::vector<FocusPointFrame> focus_point_frames;
+  std::vector<FocusPointFrame> prior_focus_point_frames;
+  std::vector<cv::Mat> all_xforms;
+  GenerateDataPoints(/* focus_point_frames_length = */ 0,
+                     /* prior_focus_point_frames_length = */ 0,
+                     &focus_point_frames, &prior_focus_point_frames);
+  constexpr int kFrameWidth = 200;
+  constexpr int kFrameHeight = 300;
+  constexpr int kCropWidth = 100;
+  constexpr int kCropHeight = 300;
+  ASSERT_FALSE(solver
+                   .ComputeCameraPath(focus_point_frames,
+                                      prior_focus_point_frames, kFrameWidth,
+                                      kFrameHeight, kCropWidth, kCropHeight,
+                                      &all_xforms)
+                   .ok());
+}
+
+}  // namespace
+}  // namespace autoflip
+}  // namespace mediapipe

mediapipe/examples/desktop/autoflip/quality/scene_camera_motion_analyzer.cc

@@ -0,0 +1,428 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/examples/desktop/autoflip/quality/scene_camera_motion_analyzer.h"
+
+#include <limits>
+
+#include "absl/memory/memory.h"
+#include "absl/strings/str_cat.h"
+#include "absl/strings/str_format.h"
+#include "mediapipe/examples/desktop/autoflip/quality/math_utils.h"
+#include "mediapipe/examples/desktop/autoflip/quality/piecewise_linear_function.h"
+#include "mediapipe/examples/desktop/autoflip/quality/utils.h"
+#include "mediapipe/framework/port/integral_types.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/timestamp.h"
+
+namespace mediapipe {
+namespace autoflip {
+
+::mediapipe::Status
+SceneCameraMotionAnalyzer::AnalyzeSceneAndPopulateFocusPointFrames(
+    const std::vector<KeyFrameInfo>& key_frame_infos,
+    const KeyFrameCropOptions& key_frame_crop_options,
+    const std::vector<KeyFrameCropResult>& key_frame_crop_results,
+    const int scene_frame_width, const int scene_frame_height,
+    const std::vector<int64>& scene_frame_timestamps,
+    SceneKeyFrameCropSummary* scene_summary,
+    std::vector<FocusPointFrame>* focus_point_frames,
+    SceneCameraMotion* scene_camera_motion) const {
+  MP_RETURN_IF_ERROR(AggregateKeyFrameResults(
+      key_frame_infos, key_frame_crop_options, key_frame_crop_results,
+      scene_frame_width, scene_frame_height, scene_summary));
+
+  const int64 scene_span_ms =
+      scene_frame_timestamps.empty()
+          ? 0
+          : scene_frame_timestamps.back() - scene_frame_timestamps.front();
+  const double scene_span_sec = TimestampDiff(scene_span_ms).Seconds();
+  SceneCameraMotion camera_motion;
+  MP_RETURN_IF_ERROR(DecideCameraMotionType(
+      key_frame_crop_options, scene_span_sec, scene_summary, &camera_motion));
+  if (scene_camera_motion != nullptr) {
+    *scene_camera_motion = camera_motion;
+  }
+
+  return PopulateFocusPointFrames(*scene_summary, camera_motion,
+                                  scene_frame_timestamps, focus_point_frames);
+}
+
+::mediapipe::Status SceneCameraMotionAnalyzer::ToUseSteadyMotion(
+    const float look_at_center_x, const float look_at_center_y,
+    const int crop_window_width, const int crop_window_height,
+    SceneKeyFrameCropSummary* scene_summary,
+    SceneCameraMotion* scene_camera_motion) const {
+  scene_summary->set_crop_window_width(crop_window_width);
+  scene_summary->set_crop_window_height(crop_window_height);
+  auto* steady_motion = scene_camera_motion->mutable_steady_motion();
+  steady_motion->set_steady_look_at_center_x(look_at_center_x);
+  steady_motion->set_steady_look_at_center_y(look_at_center_y);
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status SceneCameraMotionAnalyzer::ToUseSweepingMotion(
+    const float start_x, const float start_y, const float end_x,
+    const float end_y, const int crop_window_width,
+    const int crop_window_height, const double time_duration_in_sec,
+    SceneKeyFrameCropSummary* scene_summary,
+    SceneCameraMotion* scene_camera_motion) const {
+  auto* sweeping_motion = scene_camera_motion->mutable_sweeping_motion();
+  sweeping_motion->set_sweep_start_center_x(start_x);
+  sweeping_motion->set_sweep_start_center_y(start_y);
+  sweeping_motion->set_sweep_end_center_x(end_x);
+  sweeping_motion->set_sweep_end_center_y(end_y);
+  scene_summary->set_crop_window_width(crop_window_width);
+  scene_summary->set_crop_window_height(crop_window_height);
+  const auto sweeping_log = absl::StrFormat(
+      "Success rate %.2f is low - Camera is sweeping from (%.1f, %.1f) to "
+      "(%.1f, %.1f) in %.2f seconds.",
+      scene_summary->frame_success_rate(), start_x, start_y, end_x, end_y,
+      time_duration_in_sec);
+  VLOG(1) << sweeping_log;
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status SceneCameraMotionAnalyzer::DecideCameraMotionType(
+    const KeyFrameCropOptions& key_frame_crop_options,
+    const double scene_span_sec, SceneKeyFrameCropSummary* scene_summary,
+    SceneCameraMotion* scene_camera_motion) const {
+  RET_CHECK_GE(scene_span_sec, 0.0) << "Scene time span is negative.";
+  RET_CHECK_NE(scene_summary, nullptr) << "Scene summary is null.";
+  RET_CHECK_NE(scene_camera_motion, nullptr) << "Scene camera motion is null.";
+  const float scene_frame_center_x = scene_summary->scene_frame_width() / 2.0f;
+  const float scene_frame_center_y = scene_summary->scene_frame_height() / 2.0f;
+
+  // If no frame has any focus region, that is, the scene has no focus
+  // regions, then default to look at the center.
+  if (!scene_summary->has_salient_region()) {
+    VLOG(1) << "No focus regions - camera is set to be steady on center.";
+    MP_RETURN_IF_ERROR(ToUseSteadyMotion(
+        scene_frame_center_x, scene_frame_center_y,
+        scene_summary->crop_window_width(), scene_summary->crop_window_height(),
+        scene_summary, scene_camera_motion));
+    return ::mediapipe::OkStatus();
+  }
+
+  // Sweep across the scene when 1) success rate is too low, AND 2) the current
+  // scene is long enough.
+  if (options_.allow_sweeping() &&
+      scene_summary->frame_success_rate() <
+          options_.minimum_success_rate_for_sweeping() &&
+      scene_span_sec >= options_.minimum_scene_span_sec_for_sweeping()) {
+    float start_x = -1.0, start_y = -1.0, end_x = -1.0, end_y = -1.0;
+    if (options_.sweep_entire_frame()) {
+      if (scene_summary->crop_window_width() >
+          key_frame_crop_options.target_width()) {  // horizontal sweeping
+        start_x = 0.0f;
+        start_y = scene_frame_center_y;
+        end_x = scene_summary->scene_frame_width();
+        end_y = scene_frame_center_y;
+      } else {  // vertical sweeping
+        start_x = scene_frame_center_x;
+        start_y = 0.0f;
+        end_x = scene_frame_center_x;
+        end_y = scene_summary->scene_frame_height();
+      }
+    } else {
+      start_x = scene_summary->key_frame_center_min_x();
+      start_y = scene_summary->key_frame_center_min_y();
+      end_x = scene_summary->key_frame_center_max_x();
+      end_y = scene_summary->key_frame_center_max_y();
+    }
+    MP_RETURN_IF_ERROR(ToUseSweepingMotion(
+        start_x, start_y, end_x, end_y, key_frame_crop_options.target_width(),
+        key_frame_crop_options.target_height(), scene_span_sec, scene_summary,
+        scene_camera_motion));
+    return ::mediapipe::OkStatus();
+  }
+
+  // If scene motion is small, then look at a steady point in the scene.
+  if (scene_summary->horizontal_motion_amount() <
+          options_.motion_stabilization_threshold_percent() &&
+      scene_summary->vertical_motion_amount() <
+          options_.motion_stabilization_threshold_percent()) {
+    return DecideSteadyLookAtRegion(key_frame_crop_options, scene_summary,
+                                    scene_camera_motion);
+  }
+
+  // Otherwise, tracks the focus regions.
+  scene_camera_motion->mutable_tracking_motion();
+  return ::mediapipe::OkStatus();
+}
+
+// If there is no required focus region, looks at the middle of the center
+// range, and snaps to the scene center if close. Otherwise, look at the center
+// of the union of the required focus regions, and ensures the crop region
+// covers this union.
+::mediapipe::Status SceneCameraMotionAnalyzer::DecideSteadyLookAtRegion(
+    const KeyFrameCropOptions& key_frame_crop_options,
+    SceneKeyFrameCropSummary* scene_summary,
+    SceneCameraMotion* scene_camera_motion) const {
+  const float scene_frame_width = scene_summary->scene_frame_width();
+  const float scene_frame_height = scene_summary->scene_frame_height();
+  const int target_width = key_frame_crop_options.target_width();
+  const int target_height = key_frame_crop_options.target_height();
+  float center_x = -1, center_y = -1;
+  float crop_width = -1, crop_height = -1;
+
+  if (scene_summary->has_required_salient_region()) {
+    // Set look-at position to be the center of the union of required focus
+    // regions and the crop window size to be the maximum of this union size
+    // and the target size.
+    const auto& required_region_union =
+        scene_summary->key_frame_required_crop_region_union();
+    center_x = required_region_union.x() + required_region_union.width() / 2.0f;
+    center_y =
+        required_region_union.y() + required_region_union.height() / 2.0f;
+    crop_width = std::max(target_width, required_region_union.width());
+    crop_height = std::max(target_height, required_region_union.height());
+  } else {
+    // Set look-at position to be the middle of the center range, and the crop
+    // window size to be the target size.
+    center_x = (scene_summary->key_frame_center_min_x() +
+                scene_summary->key_frame_center_max_x()) /
+               2.0f;
+    center_y = (scene_summary->key_frame_center_min_y() +
+                scene_summary->key_frame_center_max_y()) /
+               2.0f;
+    crop_width = target_width;
+    crop_height = target_height;
+
+    // Optionally snap the look-at position to the scene frame center.
+    const float center_x_distance =
+        std::fabs(center_x - scene_frame_width / 2.0f);
+    const float center_y_distance =
+        std::fabs(center_y - scene_frame_height / 2.0f);
+    if (center_x_distance / scene_frame_width <
+        options_.snap_center_max_distance_percent()) {
+      center_x = scene_frame_width / 2.0f;
+    }
+    if (center_y_distance / scene_frame_height <
+        options_.snap_center_max_distance_percent()) {
+      center_y = scene_frame_height / 2.0f;
+    }
+  }
+
+  // Clamp the region to be inside the frame.
+  // TODO: this may not be necessary.
+  float clamped_center_x, clamped_center_y;
+  RET_CHECK(MathUtil::Clamp(crop_width / 2.0f,
+                            scene_frame_width - crop_width / 2.0f, center_x,
+                            &clamped_center_x));
+  center_x = clamped_center_x;
+  RET_CHECK(MathUtil::Clamp(crop_height / 2.0f,
+                            scene_frame_height - crop_height / 2.0f, center_y,
+                            &clamped_center_y));
+  center_y = clamped_center_y;
+
+  VLOG(1) << "Motion is small - camera is set to be steady at " << center_x
+          << ", " << center_y;
+  MP_RETURN_IF_ERROR(ToUseSteadyMotion(center_x, center_y, crop_width,
+                                       crop_height, scene_summary,
+                                       scene_camera_motion));
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status
+SceneCameraMotionAnalyzer::AddFocusPointsFromCenterTypeAndWeight(
+    const float center_x, const float center_y, const int frame_width,
+    const int frame_height, const FocusPointFrameType type, const float weight,
+    const float bound, FocusPointFrame* focus_point_frame) const {
+  RET_CHECK_NE(focus_point_frame, nullptr) << "Focus point frame is null.";
+  const float norm_x = center_x / frame_width;
+  const float norm_y = center_y / frame_height;
+  const std::vector<float> extremal_values = {0, 1};
+  if (type == TOPMOST_AND_BOTTOMMOST) {
+    for (const float extremal_value : extremal_values) {
+      auto* focus_point = focus_point_frame->add_point();
+      focus_point->set_norm_point_x(norm_x);
+      focus_point->set_norm_point_y(extremal_value);
+      focus_point->set_weight(weight);
+      focus_point->set_left(bound);
+      focus_point->set_right(bound);
+    }
+  } else if (type == LEFTMOST_AND_RIGHTMOST) {
+    for (const float extremal_value : extremal_values) {
+      auto* focus_point = focus_point_frame->add_point();
+      focus_point->set_norm_point_x(extremal_value);
+      focus_point->set_norm_point_y(norm_y);
+      focus_point->set_weight(weight);
+      focus_point->set_top(bound);
+      focus_point->set_bottom(bound);
+    }
+  } else if (type == CENTER) {
+    auto* focus_point = focus_point_frame->add_point();
+    focus_point->set_norm_point_x(norm_x);
+    focus_point->set_norm_point_y(norm_y);
+    focus_point->set_weight(weight);
+    focus_point->set_left(bound);
+    focus_point->set_right(bound);
+    focus_point->set_top(bound);
+    focus_point->set_bottom(bound);
+  } else {
+    RET_CHECK_FAIL() << absl::StrCat("Invalid FocusPointFrameType ", type);
+  }
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status SceneCameraMotionAnalyzer::PopulateFocusPointFrames(
+    const SceneKeyFrameCropSummary& scene_summary,
+    const SceneCameraMotion& scene_camera_motion,
+    const std::vector<int64>& scene_frame_timestamps,
+    std::vector<FocusPointFrame>* focus_point_frames) const {
+  RET_CHECK_NE(focus_point_frames, nullptr)
+      << "Output vector of FocusPointFrame is null.";
+
+  const int num_scene_frames = scene_frame_timestamps.size();
+  RET_CHECK_GT(num_scene_frames, 0) << "No scene frames.";
+  RET_CHECK_EQ(scene_summary.num_key_frames(),
+               scene_summary.key_frame_compact_infos_size())
+      << "Key frame compact infos has wrong size:"
+      << " num_key_frames = " << scene_summary.num_key_frames()
+      << " key_frame_compact_infos size = "
+      << scene_summary.key_frame_compact_infos_size();
+  const int scene_frame_width = scene_summary.scene_frame_width();
+  const int scene_frame_height = scene_summary.scene_frame_height();
+  RET_CHECK_GT(scene_frame_width, 0) << "Non-positive frame width.";
+  RET_CHECK_GT(scene_frame_height, 0) << "Non-positive frame height.";
+
+  FocusPointFrameType focus_point_frame_type =
+      (scene_summary.crop_window_height() == scene_frame_height)
+          ? TOPMOST_AND_BOTTOMMOST
+          : (scene_summary.crop_window_width() == scene_frame_width
+                 ? LEFTMOST_AND_RIGHTMOST
+                 : CENTER);
+  focus_point_frames->reserve(num_scene_frames);
+
+  if (scene_camera_motion.has_steady_motion()) {
+    // Camera focuses on a steady point of the scene.
+    const float center_x =
+        scene_camera_motion.steady_motion().steady_look_at_center_x();
+    const float center_y =
+        scene_camera_motion.steady_motion().steady_look_at_center_y();
+    for (int i = 0; i < num_scene_frames; ++i) {
+      FocusPointFrame focus_point_frame;
+      MP_RETURN_IF_ERROR(AddFocusPointsFromCenterTypeAndWeight(
+          center_x, center_y, scene_frame_width, scene_frame_height,
+          focus_point_frame_type, options_.maximum_salient_point_weight(),
+          options_.salient_point_bound(), &focus_point_frame));
+      focus_point_frames->push_back(focus_point_frame);
+    }
+    return ::mediapipe::OkStatus();
+  } else if (scene_camera_motion.has_sweeping_motion()) {
+    // Camera sweeps across the frame.
+    const auto& sweeping_motion = scene_camera_motion.sweeping_motion();
+    const float start_x = sweeping_motion.sweep_start_center_x();
+    const float start_y = sweeping_motion.sweep_start_center_y();
+    const float end_x = sweeping_motion.sweep_end_center_x();
+    const float end_y = sweeping_motion.sweep_end_center_y();
+    for (int i = 0; i < num_scene_frames; ++i) {
+      const float fraction =
+          num_scene_frames > 1 ? static_cast<float>(i) / (num_scene_frames - 1)
+                               : 0;
+      const float position_x = start_x * (1.0f - fraction) + end_x * fraction;
+      const float position_y = start_y * (1.0f - fraction) + end_y * fraction;
+      FocusPointFrame focus_point_frame;
+      MP_RETURN_IF_ERROR(AddFocusPointsFromCenterTypeAndWeight(
+          position_x, position_y, scene_frame_width, scene_frame_height,
+          focus_point_frame_type, options_.maximum_salient_point_weight(),
+          options_.salient_point_bound(), &focus_point_frame));
+      focus_point_frames->push_back(focus_point_frame);
+    }
+    return ::mediapipe::OkStatus();
+  } else if (scene_camera_motion.has_tracking_motion()) {
+    // Camera tracks crop regions.
+    RET_CHECK_GT(scene_summary.num_key_frames(), 0) << "No key frames.";
+    return PopulateFocusPointFramesForTracking(
+        scene_summary, focus_point_frame_type, scene_frame_timestamps,
+        focus_point_frames);
+  } else {
+    return ::mediapipe::Status(StatusCode::kInvalidArgument,
+                               "Unknown motion type.");
+  }
+}
+
+// Linearly interpolates between key frames based on the timestamps using
+// piecewise-linear functions for the crop region centers and scores. Adds one
+// focus point at the center of the interpolated crop region for each frame.
+// The weight for the focus point is proportional to the interpolated score
+// and scaled so that the maximum weight is equal to
+// maximum_focus_point_weight in the SceneCameraMotionAnalyzerOptions.
+::mediapipe::Status
+SceneCameraMotionAnalyzer::PopulateFocusPointFramesForTracking(
+    const SceneKeyFrameCropSummary& scene_summary,
+    const FocusPointFrameType focus_point_frame_type,
+    const std::vector<int64>& scene_frame_timestamps,
+    std::vector<FocusPointFrame>* focus_point_frames) const {
+  RET_CHECK_GE(scene_summary.key_frame_max_score(), 0.0)
+      << "Maximum score is negative.";
+
+  const int num_key_frames = scene_summary.num_key_frames();
+  const auto& key_frame_compact_infos = scene_summary.key_frame_compact_infos();
+  const int num_scene_frames = scene_frame_timestamps.size();
+  const int scene_frame_width = scene_summary.scene_frame_width();
+  const int scene_frame_height = scene_summary.scene_frame_height();
+
+  PiecewiseLinearFunction center_x_function, center_y_function, score_function;
+  const int64 timestamp_offset = key_frame_compact_infos[0].timestamp_ms();
+  for (int i = 0; i < num_key_frames; ++i) {
+    const float center_x = key_frame_compact_infos[i].center_x();
+    const float center_y = key_frame_compact_infos[i].center_y();
+    const float score = key_frame_compact_infos[i].score();
+    // Skips empty key frames.
+    if (center_x < 0 || center_y < 0 || score < 0) {
+      continue;
+    }
+    const double relative_timestamp =
+        key_frame_compact_infos[i].timestamp_ms() - timestamp_offset;
+    center_x_function.AddPoint(relative_timestamp, center_x);
+    center_y_function.AddPoint(relative_timestamp, center_y);
+    score_function.AddPoint(relative_timestamp, score);
+  }
+
+  double max_score = 0.0;
+  const double min_score = 1e-4;  // prevent constraints with 0 weight
+  for (int i = 0; i < num_scene_frames; ++i) {
+    const double relative_timestamp =
+        static_cast<double>(scene_frame_timestamps[i] - timestamp_offset);
+    const double center_x = center_x_function.Evaluate(relative_timestamp);
+    const double center_y = center_y_function.Evaluate(relative_timestamp);
+    const double score =
+        std::max(min_score, score_function.Evaluate(relative_timestamp));
+    max_score = std::max(max_score, score);
+    FocusPointFrame focus_point_frame;
+    MP_RETURN_IF_ERROR(AddFocusPointsFromCenterTypeAndWeight(
+        center_x, center_y, scene_frame_width, scene_frame_height,
+        focus_point_frame_type, score, options_.salient_point_bound(),
+        &focus_point_frame));
+    focus_point_frames->push_back(focus_point_frame);
+  }
+
+  // Scales weights so that maximum weight = maximum_salient_point_weight.
+  // TODO: run some experiments to find out if this is necessary.
+  max_score = std::max(max_score, min_score);
+  const double scale = options_.maximum_salient_point_weight() / max_score;
+  for (int i = 0; i < focus_point_frames->size(); ++i) {
+    for (int j = 0; j < (*focus_point_frames)[i].point_size(); ++j) {
+      auto* focus_point = (*focus_point_frames)[i].mutable_point(j);
+      focus_point->set_weight(scale * focus_point->weight());
+    }
+  }
+  return ::mediapipe::OkStatus();
+}
+
+}  // namespace autoflip
+}  // namespace mediapipe

mediapipe/examples/desktop/autoflip/quality/scene_camera_motion_analyzer.h

@@ -0,0 +1,142 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_SCENE_CAMERA_MOTION_ANALYZER_H_
+#define MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_SCENE_CAMERA_MOTION_ANALYZER_H_
+
+#include <vector>
+
+#include "mediapipe/examples/desktop/autoflip/autoflip_messages.pb.h"
+#include "mediapipe/examples/desktop/autoflip/quality/cropping.pb.h"
+#include "mediapipe/examples/desktop/autoflip/quality/focus_point.pb.h"
+#include "mediapipe/framework/port/integral_types.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+namespace autoflip {
+
+// This class does the following in order:
+// - Aggregates a key frame results to get a SceneKeyFrameCropSummary,
+// - Determines the SceneCameraMotion for the scene, and then
+// - Populates FocusPointFrames to be used as input for the retargeter.
+//
+// Upstream inputs:
+// - std::vector<KeyFrameCropInfo> key_frame_crop_infos.
+// - KeyFrameCropOptions key_frame_crop_options.
+// - std::vector<KeyFrameCropResult> key_frame_crop_results.
+// - int scene_frame_width, scene_frame_height.
+// - std::vector<int64> scene_frame_timestamps.
+//
+// Example usage:
+//   SceneCameraMotionAnalyzerOptions options;
+//   SceneCameraMotionAnalyzer analyzer(options);
+//   SceneKeyFrameCropSummary scene_summary;
+//   std::vector<FocusPointFrame> focus_point_frames;
+//   CHECK_OK(analyzer.AnalyzeScenePopulateFocusPointFrames(
+//       key_frame_crop_infos, key_frame_crop_options, key_frame_crop_results,
+//       scene_frame_width, scene_frame_height, scene_frame_timestamps,
+//       &scene_summary, &focus_point_frames));
+class SceneCameraMotionAnalyzer {
+ public:
+  SceneCameraMotionAnalyzer() = delete;
+
+  explicit SceneCameraMotionAnalyzer(const SceneCameraMotionAnalyzerOptions&
+                                         scene_camera_motion_analyzer_options)
+      : options_(scene_camera_motion_analyzer_options) {}
+
+  ~SceneCameraMotionAnalyzer() {}
+
+  // Aggregates information from KeyFrameInfos and KeyFrameCropResults into
+  // SceneKeyFrameCropSummary, and populates FocusPointFrames given scene
+  // frame timestamps. Optionally returns SceneCameraMotion.
+  ::mediapipe::Status AnalyzeSceneAndPopulateFocusPointFrames(
+      const std::vector<KeyFrameInfo>& key_frame_infos,
+      const KeyFrameCropOptions& key_frame_crop_options,
+      const std::vector<KeyFrameCropResult>& key_frame_crop_results,
+      const int scene_frame_width, const int scene_frame_height,
+      const std::vector<int64>& scene_frame_timestamps,
+      SceneKeyFrameCropSummary* scene_summary,
+      std::vector<FocusPointFrame>* focus_point_frames,
+      SceneCameraMotion* scene_camera_motion = nullptr) const;
+
+ protected:
+  // Decides SceneCameraMotion based on SceneKeyFrameCropSummary. Updates the
+  // crop window in SceneKeyFrameCropSummary in the case of steady motion.
+  ::mediapipe::Status DecideCameraMotionType(
+      const KeyFrameCropOptions& key_frame_crop_options,
+      const double scene_span_sec, SceneKeyFrameCropSummary* scene_summary,
+      SceneCameraMotion* scene_camera_motion) const;
+
+  // Populates the FocusPointFrames for each scene frame based on
+  // SceneKeyFrameCropSummary, SceneCameraMotion, and scene frame timestamps.
+  ::mediapipe::Status PopulateFocusPointFrames(
+      const SceneKeyFrameCropSummary& scene_summary,
+      const SceneCameraMotion& scene_camera_motion,
+      const std::vector<int64>& scene_frame_timestamps,
+      std::vector<FocusPointFrame>* focus_point_frames) const;
+
+ private:
+  // Decides the look-at region when camera is steady.
+  ::mediapipe::Status DecideSteadyLookAtRegion(
+      const KeyFrameCropOptions& key_frame_crop_options,
+      SceneKeyFrameCropSummary* scene_summary,
+      SceneCameraMotion* scene_camera_motion) const;
+
+  // Types of FocusPointFrames: number and placement of FocusPoint's vary.
+  enum FocusPointFrameType {
+    TOPMOST_AND_BOTTOMMOST = 1,  // (center_x, 0) and (center_x, frame_height)
+    LEFTMOST_AND_RIGHTMOST = 2,  // (0, center_y) and (frame_width, center_y)
+    CENTER = 3,                  // (center_x, center_y)
+  };
+
+  // Adds FocusPoint(s) to given FocusPointFrame given center location,
+  // frame size, FocusPointFrameType, weight, and bound.
+  ::mediapipe::Status AddFocusPointsFromCenterTypeAndWeight(
+      const float center_x, const float center_y, const int frame_width,
+      const int frame_height, const FocusPointFrameType type,
+      const float weight, const float bound,
+      FocusPointFrame* focus_point_frame) const;
+
+  // Populates the FocusPointFrames for each scene frame based on
+  // SceneKeyFrameCropSummary and scene frame timestamps in the case where
+  // camera is tracking the crop regions.
+  ::mediapipe::Status PopulateFocusPointFramesForTracking(
+      const SceneKeyFrameCropSummary& scene_summary,
+      const FocusPointFrameType focus_point_frame_type,
+      const std::vector<int64>& scene_frame_timestamps,
+      std::vector<FocusPointFrame>* focus_point_frames) const;
+
+  // Decide to use steady motion.
+  ::mediapipe::Status ToUseSteadyMotion(
+      const float look_at_center_x, const float look_at_center_y,
+      const int crop_window_width, const int crop_window_height,
+      SceneKeyFrameCropSummary* scene_summary,
+      SceneCameraMotion* scene_camera_motion) const;
+
+  // Decide to use sweeping motion.
+  ::mediapipe::Status ToUseSweepingMotion(
+      const float start_x, const float start_y, const float end_x,
+      const float end_y, const int crop_window_width,
+      const int crop_window_height, const double time_duration_in_sec,
+      SceneKeyFrameCropSummary* scene_summary,
+      SceneCameraMotion* scene_camera_motion) const;
+
+  // Scene camera motion analyzer options.
+  SceneCameraMotionAnalyzerOptions options_;
+};
+
+}  // namespace autoflip
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_SCENE_CAMERA_MOTION_ANALYZER_H_

mediapipe/examples/desktop/autoflip/quality/scene_camera_motion_analyzer_test.cc

@@ -0,0 +1,814 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/examples/desktop/autoflip/quality/scene_camera_motion_analyzer.h"
+
+#include <algorithm>
+#include <numeric>
+#include <string>
+#include <vector>
+
+#include "absl/strings/str_split.h"
+#include "mediapipe/examples/desktop/autoflip/autoflip_messages.pb.h"
+#include "mediapipe/examples/desktop/autoflip/quality/focus_point.pb.h"
+#include "mediapipe/examples/desktop/autoflip/quality/piecewise_linear_function.h"
+#include "mediapipe/framework/deps/file_path.h"
+#include "mediapipe/framework/port/file_helpers.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/port/status_matchers.h"
+
+namespace mediapipe {
+namespace autoflip {
+
+using ::testing::HasSubstr;
+
+const int kNumKeyFrames = 5;
+const int kNumSceneFrames = 30;
+
+const int64 kKeyFrameTimestampDiff = 1e6 / kNumKeyFrames;
+const int64 kSceneFrameTimestampDiff = 1e6 / kNumSceneFrames;
+// Default time span of a scene in seconds.
+const double kSceneTimeSpanSec = 1.0;
+
+const int kSceneFrameWidth = 100;
+const int kSceneFrameHeight = 100;
+
+const int kTargetWidth = 50;
+const int kTargetHeight = 50;
+
+constexpr char kCameraTrackingSceneFrameResultsFile[] =
+    "mediapipe/examples/desktop/autoflip/quality/testdata/"
+    "camera_motion_tracking_scene_frame_results.csv";
+
+// Makes a rectangle given the corner (x, y) and the size (width, height).
+Rect MakeRect(const int x, const int y, const int width, const int height) {
+  Rect rect;
+  rect.set_x(x);
+  rect.set_y(y);
+  rect.set_width(width);
+  rect.set_height(height);
+  return rect;
+}
+
+// Returns default values for KeyFrameInfos. Populates timestamps using the
+// default spacing kKeyFrameTimestampDiff starting from 0.
+std::vector<KeyFrameInfo> GetDefaultKeyFrameInfos() {
+  std::vector<KeyFrameInfo> key_frame_infos(kNumKeyFrames);
+  for (int i = 0; i < kNumKeyFrames; ++i) {
+    key_frame_infos[i].set_timestamp_ms(kKeyFrameTimestampDiff * i);
+  }
+  return key_frame_infos;
+}
+
+// Returns default values for scene frame timestamps. Populates timestamps using
+// the default spacing kSceneFrameTimestampDiff starting from 0.
+std::vector<int64> GetDefaultSceneFrameTimestamps() {
+  std::vector<int64> scene_frame_timestamps(kNumSceneFrames);
+  for (int i = 0; i < kNumSceneFrames; ++i) {
+    scene_frame_timestamps[i] = kSceneFrameTimestampDiff * i;
+  }
+  return scene_frame_timestamps;
+}
+
+// Returns default settings for KeyFrameCropOptions. Populates target size to be
+// the default target size.
+KeyFrameCropOptions GetDefaultKeyFrameCropOptions() {
+  KeyFrameCropOptions key_frame_crop_options;
+  key_frame_crop_options.set_target_width(kTargetWidth);
+  key_frame_crop_options.set_target_height(kTargetHeight);
+  return key_frame_crop_options;
+}
+
+// Returns default values for KeyFrameCropResults. Sets each frame to have
+// covered all the required regions and non-required regions, and have required
+// crop region (10, 10+20) x (10, 10+20), (full) crop region (0, 50) x (0, 50),
+// and region score 1.0.
+std::vector<KeyFrameCropResult> GetDefaultKeyFrameCropResults() {
+  std::vector<KeyFrameCropResult> key_frame_crop_results(kNumKeyFrames);
+  for (int i = 0; i < kNumKeyFrames; ++i) {
+    key_frame_crop_results[i].set_are_required_regions_covered_in_target_size(
+        true);
+    key_frame_crop_results[i].set_fraction_non_required_covered(1.0);
+    key_frame_crop_results[i].set_region_is_empty(false);
+    key_frame_crop_results[i].set_required_region_is_empty(false);
+    *(key_frame_crop_results[i].mutable_region()) = MakeRect(0, 0, 50, 50);
+    *(key_frame_crop_results[i].mutable_required_region()) =
+        MakeRect(10, 10, 20, 20);
+    key_frame_crop_results[i].set_region_score(1.0);
+  }
+  return key_frame_crop_results;
+}
+
+// Returns default settings for SceneKeyFrameCropSummary. Sets scene frame size
+// to be the default size. Sets each key frame compact info in accordance to the
+// default timestamps (using the default spacing kKeyFrameTimestampDiff starting
+// from 0), default crop regions (centered at (25, 25)), and default scores
+// (1.0). Sets center range to be [25, 25] and [25, 25]. Sets score range to be
+// [1.0, 1.0]. Sets crop window size to be (25, 25). Sets has focus region to
+// be true. Sets frame success rate to be 1.0. Sets horizontal and vertical
+// motion amount to be 0.0.
+SceneKeyFrameCropSummary GetDefaultSceneKeyFrameCropSummary() {
+  SceneKeyFrameCropSummary scene_summary;
+  scene_summary.set_scene_frame_width(kSceneFrameWidth);
+  scene_summary.set_scene_frame_height(kSceneFrameHeight);
+  scene_summary.set_num_key_frames(kNumKeyFrames);
+  for (int i = 0; i < kNumKeyFrames; ++i) {
+    auto* compact_info = scene_summary.add_key_frame_compact_infos();
+    compact_info->set_timestamp_ms(kKeyFrameTimestampDiff * i);
+    compact_info->set_center_x(25);
+    compact_info->set_center_y(25);
+    compact_info->set_score(1.0);
+  }
+  scene_summary.set_key_frame_center_min_x(25);
+  scene_summary.set_key_frame_center_max_x(25);
+  scene_summary.set_key_frame_center_min_y(25);
+  scene_summary.set_key_frame_center_max_y(25);
+  scene_summary.set_key_frame_min_score(1.0);
+  scene_summary.set_key_frame_max_score(1.0);
+  scene_summary.set_crop_window_width(25);
+  scene_summary.set_crop_window_height(25);
+  scene_summary.set_has_salient_region(true);
+  scene_summary.set_frame_success_rate(1.0);
+  scene_summary.set_horizontal_motion_amount(0.0);
+  scene_summary.set_vertical_motion_amount(0.0);
+  return scene_summary;
+}
+
+// Returns a SceneKeyFrameCropSummary with small motion. Sets crop window size
+// to default target size. Sets horizontal motion to half the threshold in the
+// options and vertical motion to 0. Sets center x range to [45, 55].
+SceneKeyFrameCropSummary GetSceneKeyFrameCropSummaryWithSmallMotion(
+    const SceneCameraMotionAnalyzerOptions& options) {
+  auto scene_summary = GetDefaultSceneKeyFrameCropSummary();
+  scene_summary.set_crop_window_width(kTargetWidth);
+  scene_summary.set_crop_window_height(kTargetHeight);
+  scene_summary.set_horizontal_motion_amount(
+      options.motion_stabilization_threshold_percent() / 2.0);
+  scene_summary.set_vertical_motion_amount(0.0);
+  scene_summary.set_key_frame_center_min_x(45);
+  scene_summary.set_key_frame_center_max_x(55);
+  return scene_summary;
+}
+
+// Testable class that allows public access to protected methods in the class.
+class TestableSceneCameraMotionAnalyzer : public SceneCameraMotionAnalyzer {
+ public:
+  explicit TestableSceneCameraMotionAnalyzer(
+      const SceneCameraMotionAnalyzerOptions&
+          scene_camera_motion_analyzer_options)
+      : SceneCameraMotionAnalyzer(scene_camera_motion_analyzer_options) {}
+  ~TestableSceneCameraMotionAnalyzer() {}
+  using SceneCameraMotionAnalyzer::DecideCameraMotionType;
+  using SceneCameraMotionAnalyzer::PopulateFocusPointFrames;
+};
+
+// Checks that DecideCameraMotionType checks that output pointers are not null.
+TEST(SceneCameraMotionAnalyzerTest, DecideCameraMotionTypeChecksOutputNotNull) {
+  SceneCameraMotionAnalyzerOptions options;
+  TestableSceneCameraMotionAnalyzer analyzer(options);
+  KeyFrameCropOptions crop_options = GetDefaultKeyFrameCropOptions();
+  SceneKeyFrameCropSummary scene_summary;
+  SceneCameraMotion camera_motion;
+  auto status = analyzer.DecideCameraMotionType(crop_options, kSceneTimeSpanSec,
+                                                nullptr, &camera_motion);
+  EXPECT_FALSE(status.ok());
+  EXPECT_THAT(status.ToString(), HasSubstr("Scene summary is null."));
+  status = analyzer.DecideCameraMotionType(crop_options, kSceneTimeSpanSec,
+                                           &scene_summary, nullptr);
+  EXPECT_FALSE(status.ok());
+  EXPECT_THAT(status.ToString(), HasSubstr("Scene camera motion is null."));
+}
+
+// Checks that DecideCameraMotionType properly handles the case where no key
+// frame has any focus region, and sets the camera motion type to steady and
+// the look-at position to the scene frame center.
+TEST(SceneCameraMotionAnalyzerTest,
+     DecideCameraMotionTypeWithoutAnyFocusRegion) {
+  SceneCameraMotionAnalyzerOptions options;
+  TestableSceneCameraMotionAnalyzer analyzer(options);
+  KeyFrameCropOptions crop_options = GetDefaultKeyFrameCropOptions();
+  auto scene_summary = GetDefaultSceneKeyFrameCropSummary();
+  scene_summary.set_has_salient_region(false);
+  SceneCameraMotion camera_motion;
+
+  MP_EXPECT_OK(analyzer.DecideCameraMotionType(crop_options, kSceneTimeSpanSec,
+                                               &scene_summary, &camera_motion));
+  EXPECT_TRUE(camera_motion.has_steady_motion());
+  const auto& steady_motion = camera_motion.steady_motion();
+  EXPECT_FLOAT_EQ(steady_motion.steady_look_at_center_x(),
+                  kSceneFrameWidth / 2.0f);
+  EXPECT_FLOAT_EQ(steady_motion.steady_look_at_center_y(),
+                  kSceneFrameHeight / 2.0f);
+}
+
+// Checks that DecideCameraMotionType properly handles the camera sweeps from
+// left to right.
+TEST(SceneCameraMotionAnalyzerTest, DecideCameraMotionTypeSweepingLeftToRight) {
+  SceneCameraMotionAnalyzerOptions options;
+  options.set_sweep_entire_frame(true);
+  TestableSceneCameraMotionAnalyzer analyzer(options);
+  auto scene_summary = GetDefaultSceneKeyFrameCropSummary();
+  scene_summary.set_frame_success_rate(
+      options.minimum_success_rate_for_sweeping() / 2.0f);
+  scene_summary.set_crop_window_width(kTargetWidth * 1.5f);  // horizontal sweep
+  scene_summary.set_crop_window_height(kTargetHeight);
+  const double time_span = options.minimum_scene_span_sec_for_sweeping() * 2.0;
+  SceneCameraMotion camera_motion;
+
+  MP_EXPECT_OK(analyzer.DecideCameraMotionType(GetDefaultKeyFrameCropOptions(),
+                                               time_span, &scene_summary,
+                                               &camera_motion));
+
+  EXPECT_TRUE(camera_motion.has_sweeping_motion());
+  const auto& sweeping_motion = camera_motion.sweeping_motion();
+  EXPECT_FLOAT_EQ(sweeping_motion.sweep_start_center_x(), 0.0f);
+  EXPECT_FLOAT_EQ(sweeping_motion.sweep_end_center_x(),
+                  static_cast<float>(kSceneFrameWidth));
+  EXPECT_FLOAT_EQ(sweeping_motion.sweep_start_center_y(),
+                  kSceneFrameHeight / 2.0f);
+  EXPECT_FLOAT_EQ(sweeping_motion.sweep_end_center_y(),
+                  kSceneFrameHeight / 2.0f);
+}
+
+// Checks that DecideCameraMotionType properly handles the camera sweeps from
+// top to bottom.
+TEST(SceneCameraMotionAnalyzerTest, DecideCameraMotionTypeSweepingTopToBottom) {
+  SceneCameraMotionAnalyzerOptions options;
+  options.set_sweep_entire_frame(true);
+  TestableSceneCameraMotionAnalyzer analyzer(options);
+  auto scene_summary = GetDefaultSceneKeyFrameCropSummary();
+  scene_summary.set_frame_success_rate(
+      options.minimum_success_rate_for_sweeping() / 2.0f);
+  scene_summary.set_crop_window_width(kTargetWidth);
+  scene_summary.set_crop_window_height(kTargetHeight * 1.5f);  // vertical sweep
+  const double time_span = options.minimum_scene_span_sec_for_sweeping() * 2.0;
+  SceneCameraMotion camera_motion;
+
+  MP_EXPECT_OK(analyzer.DecideCameraMotionType(GetDefaultKeyFrameCropOptions(),
+                                               time_span, &scene_summary,
+                                               &camera_motion));
+
+  EXPECT_TRUE(camera_motion.has_sweeping_motion());
+  const auto& sweeping_motion = camera_motion.sweeping_motion();
+  EXPECT_FLOAT_EQ(sweeping_motion.sweep_start_center_y(), 0.0f);
+  EXPECT_FLOAT_EQ(sweeping_motion.sweep_end_center_y(),
+                  static_cast<float>(kSceneFrameWidth));
+  EXPECT_FLOAT_EQ(sweeping_motion.sweep_start_center_x(),
+                  kSceneFrameWidth / 2.0f);
+  EXPECT_FLOAT_EQ(sweeping_motion.sweep_end_center_x(),
+                  kSceneFrameWidth / 2.0f);
+}
+
+// Checks that DecideCameraMotionType properly handles the camera sweeps from
+// one corner of the center range to another.
+TEST(SceneCameraMotionAnalyzerTest, DecideCameraMotionTypeSweepingCenterRange) {
+  SceneCameraMotionAnalyzerOptions options;
+  options.set_sweep_entire_frame(false);
+  TestableSceneCameraMotionAnalyzer analyzer(options);
+  auto scene_summary = GetDefaultSceneKeyFrameCropSummary();
+  scene_summary.set_frame_success_rate(
+      options.minimum_success_rate_for_sweeping() / 2.0f);
+  scene_summary.set_crop_window_width(kTargetWidth * 1.5f);
+  scene_summary.set_crop_window_height(kTargetHeight * 1.5f);
+  const double time_span = options.minimum_scene_span_sec_for_sweeping() * 2.0;
+  SceneCameraMotion camera_motion;
+
+  MP_EXPECT_OK(analyzer.DecideCameraMotionType(GetDefaultKeyFrameCropOptions(),
+                                               time_span, &scene_summary,
+                                               &camera_motion));
+
+  EXPECT_TRUE(camera_motion.has_sweeping_motion());
+  const auto& sweeping_motion = camera_motion.sweeping_motion();
+  EXPECT_FLOAT_EQ(sweeping_motion.sweep_start_center_x(),
+                  scene_summary.key_frame_center_min_x());
+  EXPECT_FLOAT_EQ(sweeping_motion.sweep_start_center_y(),
+                  scene_summary.key_frame_center_min_y());
+  EXPECT_FLOAT_EQ(sweeping_motion.sweep_end_center_x(),
+                  scene_summary.key_frame_center_max_x());
+  EXPECT_FLOAT_EQ(sweeping_motion.sweep_end_center_y(),
+                  scene_summary.key_frame_center_max_y());
+}
+
+// Checks that DecideCameraMotionType properly handles the case where motion is
+// small and there are no required focus regions.
+TEST(SceneCameraMotionAnalyzerTest,
+     DecideCameraMotionTypeSmallMotionNoRequiredFocusRegion) {
+  SceneCameraMotionAnalyzerOptions options;
+  options.set_motion_stabilization_threshold_percent(0.1);
+  options.set_snap_center_max_distance_percent(0.0);
+  TestableSceneCameraMotionAnalyzer analyzer(options);
+  auto scene_summary = GetSceneKeyFrameCropSummaryWithSmallMotion(options);
+  scene_summary.set_has_required_salient_region(false);
+  const int crop_region_center_x = 50;
+  SceneCameraMotion camera_motion;
+
+  MP_EXPECT_OK(analyzer.DecideCameraMotionType(GetDefaultKeyFrameCropOptions(),
+                                               kSceneTimeSpanSec,
+                                               &scene_summary, &camera_motion));
+  EXPECT_TRUE(camera_motion.has_steady_motion());
+  EXPECT_EQ(camera_motion.steady_motion().steady_look_at_center_x(),
+            crop_region_center_x);
+  EXPECT_EQ(scene_summary.crop_window_width(), kTargetWidth);
+  EXPECT_EQ(scene_summary.crop_window_height(), kTargetHeight);
+}
+
+// Checks that DecideCameraMotionType properly handles the case where motion is
+// small and there are required focus regions that fit in target size.
+TEST(SceneCameraMotionAnalyzerTest,
+     DecideCameraMotionTypeSmallMotionRequiredFocusRegionInTargetSize) {
+  SceneCameraMotionAnalyzerOptions options;
+  options.set_motion_stabilization_threshold_percent(0.1);
+  options.set_snap_center_max_distance_percent(0.0);
+  TestableSceneCameraMotionAnalyzer analyzer(options);
+  auto scene_summary = GetSceneKeyFrameCropSummaryWithSmallMotion(options);
+  scene_summary.set_has_required_salient_region(true);
+  *scene_summary.mutable_key_frame_required_crop_region_union() =
+      MakeRect(40, 0, 40, 10);
+  const int required_region_center_x = 60;
+  SceneCameraMotion camera_motion;
+
+  MP_EXPECT_OK(analyzer.DecideCameraMotionType(GetDefaultKeyFrameCropOptions(),
+                                               kSceneTimeSpanSec,
+                                               &scene_summary, &camera_motion));
+  EXPECT_TRUE(camera_motion.has_steady_motion());
+  EXPECT_EQ(camera_motion.steady_motion().steady_look_at_center_x(),
+            required_region_center_x);
+  EXPECT_EQ(scene_summary.crop_window_width(), 50);
+  EXPECT_EQ(scene_summary.crop_window_height(), kTargetHeight);
+}
+
+// Checks that DecideCameraMotionType properly handles the case where motion is
+// small and there are required focus regions that exceed target size.
+TEST(SceneCameraMotionAnalyzerTest,
+     DecideCameraMotionTypeSmallMotionRequiredFocusRegionExceedingTargetSize) {
+  SceneCameraMotionAnalyzerOptions options;
+  options.set_motion_stabilization_threshold_percent(0.1);
+  options.set_snap_center_max_distance_percent(0.0);
+  TestableSceneCameraMotionAnalyzer analyzer(options);
+  auto scene_summary = GetSceneKeyFrameCropSummaryWithSmallMotion(options);
+  scene_summary.set_has_required_salient_region(true);
+  *scene_summary.mutable_key_frame_required_crop_region_union() =
+      MakeRect(20, 0, 70, 10);
+  const int required_region_center_x = 55;
+  SceneCameraMotion camera_motion;
+
+  MP_EXPECT_OK(analyzer.DecideCameraMotionType(GetDefaultKeyFrameCropOptions(),
+                                               kSceneTimeSpanSec,
+                                               &scene_summary, &camera_motion));
+  EXPECT_TRUE(camera_motion.has_steady_motion());
+  EXPECT_EQ(camera_motion.steady_motion().steady_look_at_center_x(),
+            required_region_center_x);
+  EXPECT_EQ(scene_summary.crop_window_width(), 70);
+  EXPECT_EQ(scene_summary.crop_window_height(), kTargetHeight);
+}
+
+// Checks that DecideCameraMotionType properly handles the case where motion is
+// small and the middle of the key frame crop center range is close to the scene
+// frame center.
+TEST(SceneCameraMotionAnalyzerTest,
+     DecideCameraMotionTypeSmallMotionCloseToCenter) {
+  SceneCameraMotionAnalyzerOptions options;
+  options.set_motion_stabilization_threshold_percent(0.1);
+  options.set_snap_center_max_distance_percent(0.1);
+  TestableSceneCameraMotionAnalyzer analyzer(options);
+  KeyFrameCropOptions crop_options = GetDefaultKeyFrameCropOptions();
+  const float frame_center_x = kSceneFrameWidth / 2.0f;
+  auto scene_summary = GetSceneKeyFrameCropSummaryWithSmallMotion(options);
+  scene_summary.set_key_frame_center_min_x(frame_center_x - 2);
+  scene_summary.set_key_frame_center_max_x(frame_center_x);
+  SceneCameraMotion camera_motion;
+
+  MP_EXPECT_OK(analyzer.DecideCameraMotionType(crop_options, kSceneTimeSpanSec,
+                                               &scene_summary, &camera_motion));
+  EXPECT_TRUE(camera_motion.has_steady_motion());
+  EXPECT_FLOAT_EQ(camera_motion.steady_motion().steady_look_at_center_x(),
+                  frame_center_x);
+}
+
+// Checks that DecideCameraMotionType properly handles the case where motion is
+// not small, and sets the camera motion type to tracking.
+TEST(SceneCameraMotionAnalyzerTest, DecideCameraMotionTypeTracking) {
+  SceneCameraMotionAnalyzerOptions options;
+  TestableSceneCameraMotionAnalyzer analyzer(options);
+  auto scene_summary = GetDefaultSceneKeyFrameCropSummary();
+  scene_summary.set_horizontal_motion_amount(
+      options.motion_stabilization_threshold_percent() * 2.0);
+  SceneCameraMotion camera_motion;
+
+  MP_EXPECT_OK(analyzer.DecideCameraMotionType(GetDefaultKeyFrameCropOptions(),
+                                               kSceneTimeSpanSec,
+                                               &scene_summary, &camera_motion));
+  EXPECT_TRUE(camera_motion.has_tracking_motion());
+}
+
+// Checks that PopulateFocusPointFrames checks output pointer is not null.
+TEST(SceneCameraMotionAnalyzerTest,
+     PopulateFocusPointFramesChecksOutputNotNull) {
+  SceneCameraMotionAnalyzerOptions options;
+  TestableSceneCameraMotionAnalyzer analyzer(options);
+  SceneCameraMotion camera_motion;
+  const auto status = analyzer.PopulateFocusPointFrames(
+      GetDefaultSceneKeyFrameCropSummary(), camera_motion,
+      GetDefaultSceneFrameTimestamps(), nullptr);
+  EXPECT_FALSE(status.ok());
+  EXPECT_THAT(status.ToString(),
+              HasSubstr("Output vector of FocusPointFrame is null."));
+}
+
+// Checks that PopulateFocusPointFrames checks scene frames size.
+TEST(SceneCameraMotionAnalyzerTest,
+     PopulateFocusPointFramesChecksSceneFramesSize) {
+  SceneCameraMotionAnalyzerOptions options;
+  TestableSceneCameraMotionAnalyzer analyzer(options);
+  SceneCameraMotion camera_motion;
+  std::vector<int64> scene_frame_timestamps(0);
+  std::vector<FocusPointFrame> focus_point_frames;
+
+  const auto status = analyzer.PopulateFocusPointFrames(
+      GetDefaultSceneKeyFrameCropSummary(), camera_motion,
+      scene_frame_timestamps, &focus_point_frames);
+  EXPECT_FALSE(status.ok());
+  EXPECT_THAT(status.ToString(), HasSubstr("No scene frames."));
+}
+
+// Checks that PopulateFocusPointFrames handles the case of no key frames.
+TEST(SceneCameraMotionAnalyzerTest,
+     PopulateFocusPointFramesHandlesNoKeyFrames) {
+  SceneCameraMotionAnalyzerOptions options;
+  TestableSceneCameraMotionAnalyzer analyzer(options);
+  SceneCameraMotion camera_motion;
+  camera_motion.mutable_steady_motion();
+  auto scene_summary = GetDefaultSceneKeyFrameCropSummary();
+  scene_summary.set_num_key_frames(0);
+  scene_summary.clear_key_frame_compact_infos();
+  scene_summary.set_has_salient_region(false);
+  std::vector<FocusPointFrame> focus_point_frames;
+  MP_EXPECT_OK(analyzer.PopulateFocusPointFrames(
+      scene_summary, camera_motion, GetDefaultSceneFrameTimestamps(),
+      &focus_point_frames));
+}
+
+// Checks that PopulateFocusPointFrames checks KeyFrameCompactInfos has the
+// right size.
+TEST(SceneCameraMotionAnalyzerTest,
+     PopulateFocusPointFramesChecksKeyFrameCompactInfosSize) {
+  SceneCameraMotionAnalyzerOptions options;
+  TestableSceneCameraMotionAnalyzer analyzer(options);
+  SceneCameraMotion camera_motion;
+  auto scene_summary = GetDefaultSceneKeyFrameCropSummary();
+  scene_summary.set_num_key_frames(2 * kNumKeyFrames);
+  std::vector<FocusPointFrame> focus_point_frames;
+
+  const auto status = analyzer.PopulateFocusPointFrames(
+      scene_summary, camera_motion, GetDefaultSceneFrameTimestamps(),
+      &focus_point_frames);
+  EXPECT_FALSE(status.ok());
+  EXPECT_THAT(status.ToString(),
+              HasSubstr("Key frame compact infos has wrong size"));
+}
+
+// Checks that PopulateFocusPointFrames checks SceneKeyFrameCropSummary has
+// valid scene frame size.
+TEST(SceneCameraMotionAnalyzerTest,
+     PopulateFocusPointFramesChecksSceneFrameSize) {
+  SceneCameraMotionAnalyzerOptions options;
+  TestableSceneCameraMotionAnalyzer analyzer(options);
+  SceneCameraMotion camera_motion;
+  auto scene_summary = GetDefaultSceneKeyFrameCropSummary();
+  scene_summary.set_scene_frame_height(0);
+  std::vector<FocusPointFrame> focus_point_frames;
+
+  const auto status = analyzer.PopulateFocusPointFrames(
+      scene_summary, camera_motion, GetDefaultSceneFrameTimestamps(),
+      &focus_point_frames);
+  EXPECT_FALSE(status.ok());
+  EXPECT_THAT(status.ToString(), HasSubstr("Non-positive frame height."));
+}
+
+// Checks that PopulateFocusPointFrames checks camera motion type is valid.
+TEST(SceneCameraMotionAnalyzerTest,
+     PopulateFocusPointFramesChecksCameraMotionType) {
+  SceneCameraMotionAnalyzerOptions options;
+  TestableSceneCameraMotionAnalyzer analyzer(options);
+  SceneCameraMotion camera_motion;
+  camera_motion.clear_motion_type();
+  std::vector<FocusPointFrame> focus_point_frames;
+
+  const auto status = analyzer.PopulateFocusPointFrames(
+      GetDefaultSceneKeyFrameCropSummary(), camera_motion,
+      GetDefaultSceneFrameTimestamps(), &focus_point_frames);
+  EXPECT_FALSE(status.ok());
+  EXPECT_THAT(status.ToString(), HasSubstr("Unknown motion type."));
+}
+
+// Checks that PopulateFocusPointFrames properly sets FocusPointFrames when
+// camera motion type is steady.
+TEST(SceneCameraMotionAnalyzerTest, PopulateFocusPointFramesSteady) {
+  SceneCameraMotionAnalyzerOptions options;
+  TestableSceneCameraMotionAnalyzer analyzer(options);
+  SceneCameraMotion camera_motion;
+  auto* steady_motion = camera_motion.mutable_steady_motion();
+  steady_motion->set_steady_look_at_center_x(40.5);
+  steady_motion->set_steady_look_at_center_y(25);
+  std::vector<FocusPointFrame> focus_point_frames;
+
+  MP_EXPECT_OK(analyzer.PopulateFocusPointFrames(
+      GetDefaultSceneKeyFrameCropSummary(), camera_motion,
+      GetDefaultSceneFrameTimestamps(), &focus_point_frames));
+
+  EXPECT_EQ(kNumSceneFrames, focus_point_frames.size());
+  for (int i = 0; i < kNumSceneFrames; ++i) {
+    // FocusPointFrameType is CENTER.
+    EXPECT_EQ(focus_point_frames[i].point_size(), 1);
+    const auto& point = focus_point_frames[i].point(0);
+    EXPECT_FLOAT_EQ(
+        point.norm_point_x(),
+        steady_motion->steady_look_at_center_x() / kSceneFrameWidth);
+    EXPECT_FLOAT_EQ(
+        point.norm_point_y(),
+        steady_motion->steady_look_at_center_y() / kSceneFrameHeight);
+    EXPECT_FLOAT_EQ(point.weight(), options.maximum_salient_point_weight());
+  }
+}
+
+// Checks that PopulateFocusPointFrames properly sets FocusPointFrames when
+// FocusPointFrameType is TOPMOST_AND_BOTTOMMOST.
+TEST(SceneCameraMotionAnalyzerTest, PopulateFocusPointFramesTopAndBottom) {
+  SceneCameraMotionAnalyzerOptions options;
+  TestableSceneCameraMotionAnalyzer analyzer(options);
+  SceneCameraMotion camera_motion;
+  auto* steady_motion = camera_motion.mutable_steady_motion();
+  steady_motion->set_steady_look_at_center_x(40.5);
+  steady_motion->set_steady_look_at_center_y(25);
+  // Forces FocusPointFrameType to be TOPMOST_AND_BOTTOMOST.
+  auto scene_summary = GetDefaultSceneKeyFrameCropSummary();
+  scene_summary.set_crop_window_height(kSceneFrameHeight);
+  std::vector<FocusPointFrame> focus_point_frames;
+
+  MP_EXPECT_OK(analyzer.PopulateFocusPointFrames(
+      scene_summary, camera_motion, GetDefaultSceneFrameTimestamps(),
+      &focus_point_frames));
+
+  EXPECT_EQ(kNumSceneFrames, focus_point_frames.size());
+  for (int i = 0; i < kNumSceneFrames; ++i) {
+    EXPECT_EQ(focus_point_frames[i].point_size(), 2);
+    const auto& point1 = focus_point_frames[i].point(0);
+    const auto& point2 = focus_point_frames[i].point(1);
+    EXPECT_FLOAT_EQ(
+        point1.norm_point_x(),
+        steady_motion->steady_look_at_center_x() / kSceneFrameWidth);
+    EXPECT_FLOAT_EQ(point1.norm_point_y(), 0.0f);
+    EXPECT_FLOAT_EQ(
+        point2.norm_point_x(),
+        steady_motion->steady_look_at_center_x() / kSceneFrameWidth);
+    EXPECT_FLOAT_EQ(point2.norm_point_y(), 1.0f);
+    EXPECT_FLOAT_EQ(point1.weight(), options.maximum_salient_point_weight());
+    EXPECT_FLOAT_EQ(point2.weight(), options.maximum_salient_point_weight());
+  }
+}
+
+// Checks that PopulateFocusPointFrames properly sets FocusPointFrames when
+// FocusPointFrameType is LEFTMOST_AND_RIGHTMOST.
+TEST(SceneCameraMotionAnalyzerTest, PopulateFocusPointFramesLeftAndRight) {
+  SceneCameraMotionAnalyzerOptions options;
+  TestableSceneCameraMotionAnalyzer analyzer(options);
+  SceneCameraMotion camera_motion;
+  auto* steady_motion = camera_motion.mutable_steady_motion();
+  steady_motion->set_steady_look_at_center_x(40.5);
+  steady_motion->set_steady_look_at_center_y(25);
+  // Forces FocusPointFrameType to be LEFTMOST_AND_RIGHTMOST.
+  auto scene_summary = GetDefaultSceneKeyFrameCropSummary();
+  scene_summary.set_crop_window_width(kSceneFrameWidth);
+  std::vector<FocusPointFrame> focus_point_frames;
+
+  MP_EXPECT_OK(analyzer.PopulateFocusPointFrames(
+      scene_summary, camera_motion, GetDefaultSceneFrameTimestamps(),
+      &focus_point_frames));
+
+  EXPECT_EQ(kNumSceneFrames, focus_point_frames.size());
+  for (int i = 0; i < kNumSceneFrames; ++i) {
+    EXPECT_EQ(focus_point_frames[i].point_size(), 2);
+    const auto& point1 = focus_point_frames[i].point(0);
+    const auto& point2 = focus_point_frames[i].point(1);
+    EXPECT_FLOAT_EQ(point1.norm_point_x(), 0.0f);
+    EXPECT_FLOAT_EQ(
+        point1.norm_point_y(),
+        steady_motion->steady_look_at_center_y() / kSceneFrameHeight);
+    EXPECT_FLOAT_EQ(point2.norm_point_x(), 1.0f);
+    EXPECT_FLOAT_EQ(
+        point1.norm_point_y(),
+        steady_motion->steady_look_at_center_y() / kSceneFrameHeight);
+    EXPECT_FLOAT_EQ(point1.weight(), options.maximum_salient_point_weight());
+    EXPECT_FLOAT_EQ(point2.weight(), options.maximum_salient_point_weight());
+  }
+}
+
+// Checks that PopulateFocusPointFrames properly sets FocusPointFrames when
+// camera motion type is sweeping.
+TEST(SceneCameraMotionAnalyzerTest, PopulateFocusPointFramesSweeping) {
+  SceneCameraMotionAnalyzerOptions options;
+  TestableSceneCameraMotionAnalyzer analyzer(options);
+  SceneCameraMotion camera_motion;
+  auto* sweeping_motion = camera_motion.mutable_sweeping_motion();
+  sweeping_motion->set_sweep_start_center_x(5);
+  sweeping_motion->set_sweep_start_center_y(50);
+  sweeping_motion->set_sweep_end_center_x(95);
+  sweeping_motion->set_sweep_end_center_y(50);
+  const int num_frames = 10;
+  const std::vector<float> positions_x = {5,  15, 25, 35, 45,
+                                          55, 65, 75, 85, 95};
+  const std::vector<float> positions_y = {50, 50, 50, 50, 50,
+                                          50, 50, 50, 50, 50};
+  std::vector<int64> scene_frame_timestamps(num_frames);
+  std::iota(scene_frame_timestamps.begin(), scene_frame_timestamps.end(), 0);
+  std::vector<FocusPointFrame> focus_point_frames;
+
+  MP_EXPECT_OK(analyzer.PopulateFocusPointFrames(
+      GetDefaultSceneKeyFrameCropSummary(), camera_motion,
+      scene_frame_timestamps, &focus_point_frames));
+
+  EXPECT_EQ(num_frames, focus_point_frames.size());
+  for (int i = 0; i < num_frames; ++i) {
+    EXPECT_EQ(focus_point_frames[i].point_size(), 1);
+    const auto& point = focus_point_frames[i].point(0);
+    EXPECT_FLOAT_EQ(positions_x[i] / kSceneFrameWidth, point.norm_point_x());
+    EXPECT_FLOAT_EQ(positions_y[i] / kSceneFrameHeight, point.norm_point_y());
+  }
+}
+
+// Checks that PopulateFocusPointFrames checks tracking handles the case when
+// maximum score is 0.
+TEST(SceneCameraMotionAnalyzerTest,
+     PopulateFocusPointFramesTrackingHandlesZeroScore) {
+  SceneCameraMotionAnalyzerOptions options;
+  TestableSceneCameraMotionAnalyzer analyzer(options);
+  SceneCameraMotion camera_motion;
+  camera_motion.mutable_tracking_motion();
+  auto scene_summary = GetDefaultSceneKeyFrameCropSummary();
+  scene_summary.set_key_frame_max_score(0.0);
+  for (int i = 0; i < kNumKeyFrames; ++i) {
+    scene_summary.mutable_key_frame_compact_infos(i)->set_score(0.0);
+  }
+  std::vector<FocusPointFrame> focus_point_frames;
+  MP_EXPECT_OK(analyzer.PopulateFocusPointFrames(
+      scene_summary, camera_motion, GetDefaultSceneFrameTimestamps(),
+      &focus_point_frames));
+}
+
+// Checks that PopulateFocusPointFrames skips empty key frames when camera
+// motion type is tracking.
+TEST(SceneCameraMotionAnalyzerTest,
+     PopulateFocusPointFramesTrackingSkipsEmptyKeyFrames) {
+  SceneCameraMotionAnalyzerOptions options;
+  TestableSceneCameraMotionAnalyzer analyzer(options);
+  SceneCameraMotion camera_motion;
+  camera_motion.mutable_tracking_motion();
+  SceneKeyFrameCropSummary scene_summary;
+  scene_summary.set_scene_frame_width(kSceneFrameWidth);
+  scene_summary.set_scene_frame_height(kSceneFrameHeight);
+  scene_summary.set_num_key_frames(2);
+
+  // Sets first key frame to be empty and second frame to be normal.
+  const float center_x = 25.0f, center_y = 25.0f;
+  auto* first_frame_compact_info = scene_summary.add_key_frame_compact_infos();
+  first_frame_compact_info->set_center_x(-1.0);
+  auto* second_frame_compact_info = scene_summary.add_key_frame_compact_infos();
+  second_frame_compact_info->set_center_x(center_x);
+  second_frame_compact_info->set_center_y(center_y);
+  second_frame_compact_info->set_score(1.0);
+  scene_summary.set_key_frame_center_min_x(center_x);
+  scene_summary.set_key_frame_center_max_x(center_x);
+  scene_summary.set_key_frame_center_min_y(center_y);
+  scene_summary.set_key_frame_center_max_y(center_y);
+  scene_summary.set_key_frame_min_score(1.0);
+  scene_summary.set_key_frame_max_score(1.0);
+
+  // Aligns timestamps of scene frames with key frames.
+  scene_summary.mutable_key_frame_compact_infos(0)->set_timestamp_ms(10);
+  scene_summary.mutable_key_frame_compact_infos(1)->set_timestamp_ms(20);
+  std::vector<int64> scene_frame_timestamps = {10, 20};
+
+  std::vector<FocusPointFrame> focus_point_frames;
+  MP_EXPECT_OK(analyzer.PopulateFocusPointFrames(scene_summary, camera_motion,
+                                                 scene_frame_timestamps,
+                                                 &focus_point_frames));
+
+  // Both scene frames should have focus point frames based on the second key
+  // frame since the first one is empty and not used.
+  for (int i = 0; i < 2; ++i) {
+    EXPECT_EQ(focus_point_frames[i].point_size(), 1);
+    const auto& point = focus_point_frames[i].point(0);
+    EXPECT_FLOAT_EQ(point.norm_point_x(), center_x / kSceneFrameWidth);
+    EXPECT_FLOAT_EQ(point.norm_point_y(), center_y / kSceneFrameHeight);
+    EXPECT_FLOAT_EQ(point.weight(), options.maximum_salient_point_weight());
+  }
+}
+
+// Checks that PopulateFocusPointFrames properly sets FocusPointFrames when
+// camera motion type is tracking, piecewise-linearly interpolating key frame
+// centers and scores, and scaling scores so that maximum weight is equal to
+// maximum_salient_point_weight.
+TEST(SceneCameraMotionAnalyzerTest,
+     PopulateFocusPointFramesTrackingTracksKeyFrames) {
+  SceneCameraMotionAnalyzerOptions options;
+  TestableSceneCameraMotionAnalyzer analyzer(options);
+  SceneCameraMotion camera_motion;
+  camera_motion.mutable_tracking_motion();
+  auto scene_summary = GetDefaultSceneKeyFrameCropSummary();
+  const std::vector<float> centers_x = {14.0, 5.0, 40.0, 70.0, 30.0};
+  const std::vector<float> centers_y = {60.0, 50.0, 80.0, 0.0, 20.0};
+  const std::vector<float> scores = {0.1, 1.0, 2.0, 0.6, 0.9};
+  scene_summary.set_key_frame_min_score(0.1);
+  scene_summary.set_key_frame_max_score(2.0);
+  for (int i = 0; i < kNumKeyFrames; ++i) {
+    auto* compact_info = scene_summary.mutable_key_frame_compact_infos(i);
+    compact_info->set_center_x(centers_x[i]);
+    compact_info->set_center_y(centers_y[i]);
+    compact_info->set_score(scores[i]);
+  }
+
+  // Get reference scene frame results from csv file.
+  const std::string scene_frame_results_file_path =
+      mediapipe::file::JoinPath("./", kCameraTrackingSceneFrameResultsFile);
+  std::string csv_file_content;
+  MP_ASSERT_OK(mediapipe::file::GetContents(scene_frame_results_file_path,
+                                            &csv_file_content));
+  std::vector<std::string> lines = absl::StrSplit(csv_file_content, '\n');
+  std::vector<std::string> records;
+  for (const auto& line : lines) {
+    std::vector<std::string> r = absl::StrSplit(line, ',');
+    records.insert(records.end(), r.begin(), r.end());
+  }
+  CHECK_EQ(records.size(), kNumSceneFrames * 3 + 1);
+
+  std::vector<FocusPointFrame> focus_point_frames;
+  MP_EXPECT_OK(analyzer.PopulateFocusPointFrames(
+      scene_summary, camera_motion, GetDefaultSceneFrameTimestamps(),
+      &focus_point_frames));
+
+  float max_weight = 0.0;
+  const float tolerance = 1e-4;
+  for (int i = 0; i < kNumSceneFrames; ++i) {
+    EXPECT_EQ(focus_point_frames[i].point_size(), 1);
+    const auto& point = focus_point_frames[i].point(0);
+    const float expected_x = std::stof(records[i * 3]);
+    const float expected_y = std::stof(records[i * 3 + 1]);
+    const float expected_weight = std::stof(records[i * 3 + 2]);
+    EXPECT_LE(std::fabs(point.norm_point_x() - expected_x), tolerance);
+    EXPECT_LE(std::fabs(point.norm_point_y() - expected_y), tolerance);
+    EXPECT_LE(std::fabs(point.weight() - expected_weight), tolerance);
+    max_weight = std::max(max_weight, point.weight());
+  }
+  EXPECT_LE(std::fabs(max_weight - options.maximum_salient_point_weight()),
+            tolerance);
+}
+
+// Checks that AnalyzeSceneAndPopulateFocusPointFrames analyzes scene and
+// populates focus point frames.
+TEST(SceneCameraMotionAnalyzerTest, AnalyzeSceneAndPopulateFocusPointFrames) {
+  SceneCameraMotionAnalyzerOptions options;
+  SceneCameraMotionAnalyzer analyzer(options);
+  SceneKeyFrameCropSummary scene_summary;
+  std::vector<FocusPointFrame> focus_point_frames;
+
+  MP_EXPECT_OK(analyzer.AnalyzeSceneAndPopulateFocusPointFrames(
+      GetDefaultKeyFrameInfos(), GetDefaultKeyFrameCropOptions(),
+      GetDefaultKeyFrameCropResults(), kSceneFrameWidth, kSceneFrameHeight,
+      GetDefaultSceneFrameTimestamps(), &scene_summary, &focus_point_frames));
+  EXPECT_EQ(scene_summary.num_key_frames(), kNumKeyFrames);
+  EXPECT_EQ(focus_point_frames.size(), kNumSceneFrames);
+}
+
+// Checks that AnalyzeSceneAndPopulateFocusPointFrames optionally returns
+// scene camera motion.
+TEST(SceneCameraMotionAnalyzerTest,
+     AnalyzeSceneAndPopulateFocusPointFramesReturnsSceneCameraMotion) {
+  SceneCameraMotionAnalyzerOptions options;
+  SceneCameraMotionAnalyzer analyzer(options);
+  SceneKeyFrameCropSummary scene_summary;
+  std::vector<FocusPointFrame> focus_point_frames;
+  SceneCameraMotion scene_camera_motion;
+
+  MP_EXPECT_OK(analyzer.AnalyzeSceneAndPopulateFocusPointFrames(
+      GetDefaultKeyFrameInfos(), GetDefaultKeyFrameCropOptions(),
+      GetDefaultKeyFrameCropResults(), kSceneFrameWidth, kSceneFrameHeight,
+      GetDefaultSceneFrameTimestamps(), &scene_summary, &focus_point_frames,
+      &scene_camera_motion));
+  EXPECT_TRUE(scene_camera_motion.has_steady_motion());
+}
+
+}  // namespace autoflip
+}  // namespace mediapipe

mediapipe/examples/desktop/autoflip/quality/scene_cropper.cc

@@ -0,0 +1,84 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/examples/desktop/autoflip/quality/scene_cropper.h"
+
+#include "absl/memory/memory.h"
+#include "mediapipe/examples/desktop/autoflip/quality/polynomial_regression_path_solver.h"
+#include "mediapipe/examples/desktop/autoflip/quality/utils.h"
+#include "mediapipe/framework/port/opencv_core_inc.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+namespace autoflip {
+
+::mediapipe::Status SceneCropper::CropFrames(
+    const SceneKeyFrameCropSummary& scene_summary,
+    const std::vector<cv::Mat>& scene_frames,
+    const std::vector<FocusPointFrame>& focus_point_frames,
+    const std::vector<FocusPointFrame>& prior_focus_point_frames,
+    std::vector<cv::Mat>* cropped_frames) const {
+  RET_CHECK_NE(cropped_frames, nullptr) << "Output cropped frames is null.";
+
+  const int num_scene_frames = scene_frames.size();
+  RET_CHECK_GT(num_scene_frames, 0) << "No scene frames.";
+  RET_CHECK_EQ(focus_point_frames.size(), num_scene_frames)
+      << "Wrong size of FocusPointFrames.";
+
+  const int frame_width = scene_summary.scene_frame_width();
+  const int frame_height = scene_summary.scene_frame_height();
+  const int crop_width = scene_summary.crop_window_width();
+  const int crop_height = scene_summary.crop_window_height();
+  RET_CHECK_GT(crop_width, 0) << "Crop width is non-positive.";
+  RET_CHECK_GT(crop_height, 0) << "Crop height is non-positive.";
+  RET_CHECK_LE(crop_width, frame_width) << "Crop width exceeds frame width.";
+  RET_CHECK_LE(crop_height, frame_height)
+      << "Crop height exceeds frame height.";
+
+  // Computes transforms.
+  std::vector<cv::Mat> all_xforms;
+
+  PolynomialRegressionPathSolver solver;
+  RET_CHECK_OK(solver.ComputeCameraPath(
+      focus_point_frames, prior_focus_point_frames, frame_width, frame_height,
+      crop_width, crop_height, &all_xforms));
+
+  const int num_prior = prior_focus_point_frames.size();
+  std::vector<cv::Mat> scene_frame_xforms(all_xforms.begin() + num_prior,
+                                          all_xforms.end());
+
+  // Convert the matrix from center-aligned to upper-left aligned.
+  for (cv::Mat& xform : scene_frame_xforms) {
+    cv::Mat affine_opencv = cv::Mat::eye(2, 3, CV_32FC1);
+    affine_opencv.at<float>(0, 2) =
+        -(xform.at<float>(0, 2) + frame_width / 2 - crop_width / 2);
+    affine_opencv.at<float>(1, 2) =
+        -(xform.at<float>(1, 2) + frame_height / 2 - crop_height / 2);
+    xform = affine_opencv;
+  }
+
+  // Prepares cropped frames.
+  cropped_frames->resize(num_scene_frames);
+  for (int i = 0; i < num_scene_frames; ++i) {
+    (*cropped_frames)[i] =
+        cv::Mat::zeros(crop_height, crop_width, scene_frames[i].type());
+  }
+
+  return AffineRetarget(cv::Size(crop_width, crop_height), scene_frames,
+                        scene_frame_xforms, cropped_frames);
+}
+
+}  // namespace autoflip
+}  // namespace mediapipe

mediapipe/examples/desktop/autoflip/quality/scene_cropper.h

@@ -0,0 +1,65 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_SCENE_CROPPER_H_
+#define MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_SCENE_CROPPER_H_
+
+#include <memory>
+#include <vector>
+
+#include "mediapipe/examples/desktop/autoflip/quality/cropping.pb.h"
+#include "mediapipe/examples/desktop/autoflip/quality/focus_point.pb.h"
+#include "mediapipe/framework/port/opencv_core_inc.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+namespace autoflip {
+
+// This class is a thin wrapper around the Retargeter class to crop a collection
+// of scene frames given SceneKeyFrameCropSummary and their FocusPointFrames.
+//
+// Upstream inputs:
+// - SceneKeyFrameCropSummary scene_summary.
+// - std::vector<FocusPointFrame> focus_point_frames.
+// - std::vector<FocusPointFrame> prior_focus_point_frames.
+// - std::vector<cv::Mat> scene_frames;
+//
+// Example usage:
+//   SceneCropperOptions scene_cropper_options;
+//   SceneCropper scene_cropper(scene_cropper_options);
+//   std::vector<cv::Mat> cropped_frames;
+//   CHECK_OK(scene_cropper.CropFrames(
+//       scene_summary, scene_frames, focus_point_frames,
+//       prior_focus_point_frames, &cropped_frames));
+class SceneCropper {
+ public:
+  SceneCropper() {}
+  ~SceneCropper() {}
+
+  // Crops scene frames given SceneKeyFrameCropSummary, FocusPointFrames, and
+  // any prior FocusPointFrames (to ensure smoothness when there was no actual
+  // scene change).
+  ::mediapipe::Status CropFrames(
+      const SceneKeyFrameCropSummary& scene_summary,
+      const std::vector<cv::Mat>& scene_frames,
+      const std::vector<FocusPointFrame>& focus_point_frames,
+      const std::vector<FocusPointFrame>& prior_focus_point_frames,
+      std::vector<cv::Mat>* cropped_frames) const;
+};
+
+}  // namespace autoflip
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_SCENE_CROPPER_H_

mediapipe/examples/desktop/autoflip/quality/scene_cropper_test.cc

@@ -0,0 +1,164 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/examples/desktop/autoflip/quality/scene_cropper.h"
+
+#include "mediapipe/examples/desktop/autoflip/quality/focus_point.pb.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/opencv_core_inc.h"
+#include "mediapipe/framework/port/status_matchers.h"
+
+namespace mediapipe {
+namespace autoflip {
+
+using testing::HasSubstr;
+
+const int kCropWidth = 90;
+const int kCropHeight = 160;
+
+const int kSceneWidth = 320;
+const int kSceneHeight = 180;
+
+const int kNumSceneFrames = 30;
+
+// Returns default values for SceneKeyFrameCropSummary. Sets scene size and crop
+// window size from default values.
+SceneKeyFrameCropSummary GetDefaultSceneKeyFrameCropSummary() {
+  SceneKeyFrameCropSummary scene_summary;
+  scene_summary.set_scene_frame_width(kSceneWidth);
+  scene_summary.set_scene_frame_height(kSceneHeight);
+  scene_summary.set_crop_window_width(kCropWidth);
+  scene_summary.set_crop_window_height(kCropHeight);
+  return scene_summary;
+}
+
+// Returns default values for scene frames of size kNumSceneFrames. Stes each
+// frame to be solid red color at default scene size.
+std::vector<cv::Mat> GetDefaultSceneFrames() {
+  std::vector<cv::Mat> scene_frames(kNumSceneFrames);
+  for (int i = 0; i < kNumSceneFrames; ++i) {
+    scene_frames[i] = cv::Mat(kSceneHeight, kSceneWidth, CV_8UC3);
+    scene_frames[i] = cv::Scalar(255, 0, 0);
+  }
+  return scene_frames;
+}
+
+// Makes a vector of FocusPointFrames given size. Stes each FocusPointFrame
+// to have one FocusPoint at the center of the frame.
+std::vector<FocusPointFrame> GetFocusPointFrames(const int num_frames) {
+  std::vector<FocusPointFrame> focus_point_frames(num_frames);
+  for (int i = 0; i < num_frames; ++i) {
+    auto* point = focus_point_frames[i].add_point();
+    point->set_norm_point_x(0.5);
+    point->set_norm_point_y(0.5);
+  }
+  return focus_point_frames;
+}
+// Returns default values for FocusPointFrames of size kNumSceneFrames.
+std::vector<FocusPointFrame> GetDefaultFocusPointFrames() {
+  return GetFocusPointFrames(kNumSceneFrames);
+}
+
+// Checks that CropFrames checks output pointer is not null.
+TEST(SceneCropperTest, CropFramesChecksOutputNotNull) {
+  SceneCropper scene_cropper;
+  const auto status = scene_cropper.CropFrames(
+      GetDefaultSceneKeyFrameCropSummary(), GetDefaultSceneFrames(),
+      GetDefaultFocusPointFrames(), GetFocusPointFrames(0), nullptr);
+  EXPECT_FALSE(status.ok());
+  EXPECT_THAT(status.ToString(), HasSubstr("Output cropped frames is null."));
+}
+
+// Checks that CropFrames checks that scene frames size is positive.
+TEST(SceneCropperTest, CropFramesChecksSceneFramesSize) {
+  SceneCropper scene_cropper;
+  std::vector<cv::Mat> scene_frames(0);
+  std::vector<cv::Mat> cropped_frames;
+  const auto status = scene_cropper.CropFrames(
+      GetDefaultSceneKeyFrameCropSummary(), scene_frames,
+      GetDefaultFocusPointFrames(), GetFocusPointFrames(0), &cropped_frames);
+  EXPECT_FALSE(status.ok());
+  EXPECT_THAT(status.ToString(), HasSubstr("No scene frames."));
+}
+
+// Checks that CropFrames checks that FocusPointFrames has the right size.
+TEST(SceneCropperTest, CropFramesChecksFocusPointFramesSize) {
+  SceneCropper scene_cropper;
+  std::vector<cv::Mat> cropped_frames;
+  const auto status = scene_cropper.CropFrames(
+      GetDefaultSceneKeyFrameCropSummary(), GetDefaultSceneFrames(),
+      GetFocusPointFrames(kNumSceneFrames - 1), GetFocusPointFrames(0),
+      &cropped_frames);
+  EXPECT_FALSE(status.ok());
+  EXPECT_THAT(status.ToString(), HasSubstr("Wrong size of FocusPointFrames"));
+}
+
+// Checks that CropFrames checks crop size is positive.
+TEST(SceneCropperTest, CropFramesChecksCropSizePositive) {
+  auto scene_summary = GetDefaultSceneKeyFrameCropSummary();
+  scene_summary.set_crop_window_width(-1);
+  SceneCropper scene_cropper;
+  std::vector<cv::Mat> cropped_frames;
+  const auto status = scene_cropper.CropFrames(
+      scene_summary, GetDefaultSceneFrames(), GetDefaultFocusPointFrames(),
+      GetFocusPointFrames(0), &cropped_frames);
+  EXPECT_FALSE(status.ok());
+  EXPECT_THAT(status.ToString(), HasSubstr("Crop width is non-positive."));
+}
+
+// Checks that CropFrames checks that crop size does not exceed frame size.
+TEST(SceneCropperTest, InitializesRetargeterChecksCropSizeNotExceedFrameSize) {
+  auto scene_summary = GetDefaultSceneKeyFrameCropSummary();
+  scene_summary.set_crop_window_height(kSceneHeight + 1);
+  SceneCropper scene_cropper;
+  std::vector<cv::Mat> cropped_frames;
+  const auto status = scene_cropper.CropFrames(
+      scene_summary, GetDefaultSceneFrames(), GetDefaultFocusPointFrames(),
+      GetFocusPointFrames(0), &cropped_frames);
+  EXPECT_FALSE(status.ok());
+  EXPECT_THAT(status.ToString(),
+              HasSubstr("Crop height exceeds frame height."));
+}
+
+// Checks that CropFrames works when there are not any prior FocusPointFrames.
+TEST(SceneCropperTest, CropFramesWorksWithoutPriorFocusPointFrames) {
+  SceneCropper scene_cropper;
+  std::vector<cv::Mat> cropped_frames;
+  MP_ASSERT_OK(scene_cropper.CropFrames(
+      GetDefaultSceneKeyFrameCropSummary(), GetDefaultSceneFrames(),
+      GetDefaultFocusPointFrames(), GetFocusPointFrames(0), &cropped_frames));
+  ASSERT_EQ(cropped_frames.size(), kNumSceneFrames);
+  for (int i = 0; i < kNumSceneFrames; ++i) {
+    EXPECT_EQ(cropped_frames[i].rows, kCropHeight);
+    EXPECT_EQ(cropped_frames[i].cols, kCropWidth);
+  }
+}
+
+// Checks that CropFrames works when there are prior FocusPointFrames.
+TEST(SceneCropperTest, CropFramesWorksWithPriorFocusPointFrames) {
+  SceneCropper scene_cropper;
+  std::vector<cv::Mat> cropped_frames;
+  MP_EXPECT_OK(scene_cropper.CropFrames(
+      GetDefaultSceneKeyFrameCropSummary(), GetDefaultSceneFrames(),
+      GetDefaultFocusPointFrames(), GetFocusPointFrames(3), &cropped_frames));
+  EXPECT_EQ(cropped_frames.size(), kNumSceneFrames);
+  for (int i = 0; i < kNumSceneFrames; ++i) {
+    EXPECT_EQ(cropped_frames[i].rows, kCropHeight);
+    EXPECT_EQ(cropped_frames[i].cols, kCropWidth);
+  }
+}
+
+}  // namespace autoflip
+}  // namespace mediapipe

mediapipe/examples/desktop/autoflip/quality/scene_cropping_viz.cc

@@ -0,0 +1,186 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/examples/desktop/autoflip/quality/scene_cropping_viz.h"
+
+#include <memory>
+
+#include "absl/memory/memory.h"
+#include "mediapipe/examples/desktop/autoflip/autoflip_messages.pb.h"
+#include "mediapipe/examples/desktop/autoflip/quality/cropping.pb.h"
+#include "mediapipe/examples/desktop/autoflip/quality/focus_point.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/image_format.pb.h"
+#include "mediapipe/framework/formats/image_frame.h"
+#include "mediapipe/framework/formats/image_frame_opencv.h"
+#include "mediapipe/framework/port/opencv_core_inc.h"
+#include "mediapipe/framework/port/opencv_imgproc_inc.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+namespace autoflip {
+
+// Colors for focus signal sources.
+const cv::Scalar kCyan =
+    cv::Scalar(0.0, 255.0, 255.0);  // brain object detector
+const cv::Scalar kMagenta = cv::Scalar(255.0, 0.0, 255.0);        // motion
+const cv::Scalar kYellow = cv::Scalar(255.0, 255.0, 0.0);         // fg ocr
+const cv::Scalar kLightYellow = cv::Scalar(255.0, 250.0, 205.0);  // bg ocr
+const cv::Scalar kRed = cv::Scalar(255.0, 0.0, 0.0);              // logo
+const cv::Scalar kGreen = cv::Scalar(0.0, 255.0, 0.0);            // face
+const cv::Scalar kBlue =
+    cv::Scalar(0.0, 0.0, 255.0);  // creatism saliency model
+const cv::Scalar kOrange =
+    cv::Scalar(255.0, 165.0, 0.0);  // ica object detector
+const cv::Scalar kWhite = cv::Scalar(255.0, 255.0, 255.0);  // others
+
+::mediapipe::Status DrawDetectionsAndCropRegions(
+    const std::vector<cv::Mat>& scene_frames,
+    const std::vector<bool>& is_key_frames,
+    const std::vector<KeyFrameInfo>& key_frame_infos,
+    const std::vector<KeyFrameCropResult>& key_frame_crop_results,
+    const ImageFormat::Format image_format,
+    std::vector<std::unique_ptr<ImageFrame>>* viz_frames) {
+  RET_CHECK(viz_frames) << "Output viz frames is null.";
+  viz_frames->clear();
+  const int num_frames = scene_frames.size();
+
+  std::pair<cv::Point, cv::Point> crop_corners;
+  std::vector<std::pair<cv::Point, cv::Point>> region_corners;
+  std::vector<cv::Scalar> region_colors;
+  auto RectToCvPoints =
+      [](const Rect& rect) -> std::pair<cv::Point, cv::Point> {
+    return std::make_pair(
+        cv::Point(rect.x(), rect.y()),
+        cv::Point(rect.x() + rect.width(), rect.y() + rect.height()));
+  };
+
+  int key_frame_idx = 0;
+  for (int i = 0; i < num_frames; ++i) {
+    const auto& scene_frame = scene_frames[i];
+    auto viz_frame = absl::make_unique<ImageFrame>(
+        image_format, scene_frame.cols, scene_frame.rows);
+    cv::Mat viz_mat = formats::MatView(viz_frame.get());
+    scene_frame.copyTo(viz_mat);
+
+    if (is_key_frames[i]) {
+      const auto& bbox = key_frame_crop_results[key_frame_idx].region();
+      crop_corners = RectToCvPoints(bbox);
+      region_corners.clear();
+      region_colors.clear();
+      const auto& detections = key_frame_infos[key_frame_idx].detections();
+      for (int j = 0; j < detections.detections_size(); ++j) {
+        const auto& detection = detections.detections(j);
+        const auto corners = RectToCvPoints(detection.location());
+        region_corners.push_back(corners);
+        if (detection.signal_type().has_standard()) {
+          switch (detection.signal_type().standard()) {
+            case SignalType::FACE_FULL:
+            case SignalType::FACE_LANDMARK:
+            case SignalType::FACE_ALL_LANDMARKS:
+            case SignalType::FACE_CORE_LANDMARKS:
+              region_colors.push_back(kGreen);
+              break;
+            case SignalType::HUMAN:
+              region_colors.push_back(kLightYellow);
+              break;
+            case SignalType::CAR:
+              region_colors.push_back(kMagenta);
+              break;
+            case SignalType::PET:
+              region_colors.push_back(kYellow);
+              break;
+            case SignalType::OBJECT:
+              region_colors.push_back(kCyan);
+              break;
+            case SignalType::MOTION:
+            case SignalType::TEXT:
+            case SignalType::LOGO:
+              region_colors.push_back(kRed);
+              break;
+            case SignalType::USER_HINT:
+            default:
+              region_colors.push_back(kWhite);
+              break;
+          }
+        } else {
+          // For the case where "custom" signal type is used.
+          region_colors.push_back(kWhite);
+        }
+      }
+      key_frame_idx++;
+    }
+
+    cv::rectangle(viz_mat, crop_corners.first, crop_corners.second, kGreen, 4);
+    for (int j = 0; j < region_corners.size(); ++j) {
+      cv::rectangle(viz_mat, region_corners[j].first, region_corners[j].second,
+                    region_colors[j], 2);
+    }
+    viz_frames->push_back(std::move(viz_frame));
+  }
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status DrawFocusPointAndCropWindow(
+    const std::vector<cv::Mat>& scene_frames,
+    const std::vector<FocusPointFrame>& focus_point_frames,
+    const float overlay_opacity, const int crop_window_width,
+    const int crop_window_height, const ImageFormat::Format image_format,
+    std::vector<std::unique_ptr<ImageFrame>>* viz_frames) {
+  RET_CHECK(viz_frames) << "Output viz frames is null.";
+  viz_frames->clear();
+  const int num_frames = scene_frames.size();
+  RET_CHECK_GT(crop_window_width, 0) << "Crop window width is non-positive.";
+  RET_CHECK_GT(crop_window_height, 0) << "Crop window height is non-positive.";
+  const int half_width = crop_window_width / 2;
+  const int half_height = crop_window_height / 2;
+
+  for (int i = 0; i < num_frames; ++i) {
+    const auto& scene_frame = scene_frames[i];
+    auto viz_frame = absl::make_unique<ImageFrame>(
+        image_format, scene_frame.cols, scene_frame.rows);
+    cv::Mat darkened = formats::MatView(viz_frame.get());
+    scene_frame.copyTo(darkened);
+    cv::Mat viz_mat = darkened.clone();
+
+    // Darken the background.
+    cv::Mat overlay = cv::Mat::zeros(darkened.size(), darkened.type());
+    cv::addWeighted(overlay, overlay_opacity, darkened, 1 - overlay_opacity, 0,
+                    darkened);
+
+    if (focus_point_frames[i].point_size() > 0) {
+      float center_x = 0.0f, center_y = 0.0f;
+      for (int j = 0; j < focus_point_frames[i].point_size(); ++j) {
+        const auto& point = focus_point_frames[i].point(j);
+        const int x = point.norm_point_x() * scene_frame.cols;
+        const int y = point.norm_point_y() * scene_frame.rows;
+        cv::circle(viz_mat, cv::Point(x, y), 3, kRed, CV_FILLED);
+        center_x += x;
+        center_y += y;
+      }
+      center_x /= focus_point_frames[i].point_size();
+      center_y /= focus_point_frames[i].point_size();
+      cv::Point min_corner(center_x - half_width, center_y - half_height);
+      cv::Point max_corner(center_x + half_width, center_y + half_height);
+      viz_mat(cv::Rect(min_corner, max_corner))
+          .copyTo(darkened(cv::Rect(min_corner, max_corner)));
+    }
+    viz_frames->push_back(std::move(viz_frame));
+  }
+  return ::mediapipe::OkStatus();
+}
+
+}  // namespace autoflip
+}  // namespace mediapipe

mediapipe/examples/desktop/autoflip/quality/scene_cropping_viz.h

@@ -0,0 +1,61 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/examples/desktop/autoflip/quality/focus_point.pb.h"
+#ifndef MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_SCENE_CROPPING_VIZ_H_
+#define MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_SCENE_CROPPING_VIZ_H_
+
+#include <vector>
+
+#include "mediapipe/examples/desktop/autoflip/quality/cropping.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/image_format.pb.h"
+#include "mediapipe/framework/formats/image_frame.h"
+#include "mediapipe/framework/port/opencv_core_inc.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+namespace autoflip {
+
+// Draws the detections and crop regions on the scene frame. To make
+// visualization smoother, applies piecewise-constant interpolation on non-key
+// frames. This helps visualize the inputs to and outputs from the
+// FrameCropRegionComputer. Uses thick green for computed crop regions. Uses
+// different colors for different focus signals, faces are green, motion is
+// magenta, logos are red, ocrs are yellow (foreground) and light yellow
+// (background), brain objects are cyan, ica objects are orange, and the rest
+// are white.
+::mediapipe::Status DrawDetectionsAndCropRegions(
+    const std::vector<cv::Mat>& scene_frames,
+    const std::vector<bool>& is_key_frames,
+    const std::vector<KeyFrameInfo>& key_frame_infos,
+    const std::vector<KeyFrameCropResult>& key_frame_crop_results,
+    const mediapipe::ImageFormat::Format image_format,
+    std::vector<std::unique_ptr<ImageFrame>>* viz_frames);
+
+// Draws the focus point from the given FocusPointFrame and the crop window
+// centered around it on the scene frame in red. This helps visualize the input
+// to the retargeter.
+::mediapipe::Status DrawFocusPointAndCropWindow(
+    const std::vector<cv::Mat>& scene_frames,
+    const std::vector<FocusPointFrame>& focus_point_frames,
+    const float overlay_opacity, const int crop_window_width,
+    const int crop_window_height,
+    const mediapipe::ImageFormat::Format image_format,
+    std::vector<std::unique_ptr<ImageFrame>>* viz_frames);
+
+}  // namespace autoflip
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_SCENE_CROPPING_VIZ_H_

mediapipe/examples/desktop/autoflip/quality/testdata/BUILD

@@ -0,0 +1,19 @@
+# Copyright 2019 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+licenses(["notice"])  # Apache 2.0
+
+package(default_visibility = ["//visibility:public"])
+
+exports_files(glob(["*"]))

mediapipe/examples/desktop/autoflip/quality/testdata/camera_motion_tracking_scene_frame_results.csv

@@ -0,0 +1,30 @@
+0.14,0.6,5.00005
+0.125,0.583333,12.5
+0.11,0.566667,20
+0.0950004,0.55,27.5
+0.0800006,0.533334,35
+0.0650008,0.516668,42.5
+0.0500009,0.500001,50
+0.108329,0.549996,58.3333
+0.166662,0.599996,66.6667
+0.224995,0.649996,75
+0.283327,0.699995,83.3333
+0.34166,0.749995,91.6667
+0.399993,0.799994,100
+0.449994,0.666684,88.3357
+0.499993,0.533352,76.6691
+0.549993,0.40002,65.0024
+0.599992,0.266688,53.3357
+0.649992,0.133356,41.6691
+0.699991,2.40E-05,30.0024
+0.633346,0.033327,32.4999
+0.56668,0.06666,34.9999
+0.500014,0.099993,37.4999
+0.433348,0.133326,39.9998
+0.366682,0.166659,42.4998
+0.300016,0.199992,44.9999
+0.3,0.2,45.0005
+0.3,0.2,45.0005
+0.3,0.2,45.0005
+0.3,0.2,45.0005
+0.3,0.2,45.0005

mediapipe/examples/desktop/autoflip/quality/utils.cc

@@ -0,0 +1,451 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/examples/desktop/autoflip/quality/utils.h"
+
+#include <math.h>
+
+#include <algorithm>
+#include <utility>
+
+#include "absl/memory/memory.h"
+#include "mediapipe/examples/desktop/autoflip/quality/math_utils.h"
+#include "mediapipe/framework/port/opencv_imgproc_inc.h"
+#include "mediapipe/framework/port/ret_check.h"
+
+namespace mediapipe {
+namespace autoflip {
+namespace {
+
+// Returns true if the first pair should be considered greater than the second.
+// This is used to sort detections by scores (from high to low).
+bool PairCompare(const std::pair<float, int>& pair1,
+                 const std::pair<float, int>& pair2) {
+  return pair1.first > pair2.first;
+}
+
+}  // namespace
+
+template <typename T>
+void ScaleRect(const T& original_location, const double scale_x,
+               const double scale_y, Rect* scaled_location) {
+  scaled_location->set_x(round(original_location.x() * scale_x));
+  scaled_location->set_y(round(original_location.y() * scale_y));
+  scaled_location->set_width(round(original_location.width() * scale_x));
+  scaled_location->set_height(round(original_location.height() * scale_y));
+}
+template void ScaleRect<Rect>(const Rect&, const double, const double, Rect*);
+template void ScaleRect<RectF>(const RectF&, const double, const double, Rect*);
+
+void NormalizedRectToRect(const RectF& normalized_location, const int width,
+                          const int height, Rect* location) {
+  ScaleRect(normalized_location, width, height, location);
+}
+
+::mediapipe::Status ClampRect(const int width, const int height,
+                              Rect* location) {
+  return ClampRect(0, 0, width, height, location);
+}
+
+::mediapipe::Status ClampRect(const int x0, const int y0, const int x1,
+                              const int y1, Rect* location) {
+  RET_CHECK(!(location->x() >= x1 || location->x() + location->width() <= x0 ||
+              location->y() >= y1 || location->y() + location->height() <= y0));
+
+  int clamped_left, clamped_right, clamped_top, clamped_bottom;
+  RET_CHECK(MathUtil::Clamp(x0, x1, location->x(), &clamped_left));
+  RET_CHECK(MathUtil::Clamp(x0, x1, location->x() + location->width(),
+                            &clamped_right));
+  RET_CHECK(MathUtil::Clamp(y0, y1, location->y(), &clamped_top));
+  RET_CHECK(MathUtil::Clamp(y0, y1, location->y() + location->height(),
+                            &clamped_bottom));
+  location->set_x(clamped_left);
+  location->set_y(clamped_top);
+  location->set_width(std::max(0, clamped_right - clamped_left));
+  location->set_height(std::max(0, clamped_bottom - clamped_top));
+  return ::mediapipe::OkStatus();
+}
+
+void RectUnion(const Rect& rect_to_add, Rect* rect) {
+  const int x1 = std::min(rect->x(), rect_to_add.x());
+  const int y1 = std::min(rect->y(), rect_to_add.y());
+  const int x2 = std::max(rect->x() + rect->width(),
+                          rect_to_add.x() + rect_to_add.width());
+  const int y2 = std::max(rect->y() + rect->height(),
+                          rect_to_add.y() + rect_to_add.height());
+  rect->set_x(x1);
+  rect->set_y(y1);
+  rect->set_width(x2 - x1);
+  rect->set_height(y2 - y1);
+}
+
+::mediapipe::Status PackKeyFrameInfo(const int64 frame_timestamp_ms,
+                                     const DetectionSet& detections,
+                                     const int original_frame_width,
+                                     const int original_frame_height,
+                                     const int feature_frame_width,
+                                     const int feature_frame_height,
+                                     KeyFrameInfo* key_frame_info) {
+  RET_CHECK(key_frame_info != nullptr) << "KeyFrameInfo is null";
+  RET_CHECK(original_frame_width > 0 && original_frame_height > 0 &&
+            feature_frame_width > 0 && feature_frame_height > 0)
+      << "Invalid frame size.";
+
+  const double scale_x =
+      static_cast<double>(original_frame_width) / feature_frame_width;
+  const double scale_y =
+      static_cast<double>(original_frame_height) / feature_frame_height;
+
+  key_frame_info->set_timestamp_ms(frame_timestamp_ms);
+
+  // Scales detections and filter out the ones with no bounding boxes.
+  auto* processed_detections = key_frame_info->mutable_detections();
+  for (const auto& original_detection : detections.detections()) {
+    bool has_valid_location = true;
+    Rect location;
+    if (original_detection.has_location_normalized()) {
+      NormalizedRectToRect(original_detection.location_normalized(),
+                           original_frame_width, original_frame_height,
+                           &location);
+    } else if (original_detection.has_location()) {
+      ScaleRect(original_detection.location(), scale_x, scale_y, &location);
+    } else {
+      has_valid_location = false;
+      LOG(ERROR) << "Detection missing a bounding box, skipped.";
+    }
+    if (has_valid_location) {
+      auto* detection = processed_detections->add_detections();
+      *detection = original_detection;
+      RET_CHECK_OK(
+          ClampRect(original_frame_width, original_frame_height, &location));
+      *(detection->mutable_location()) = location;
+    }
+  }
+
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status SortDetections(
+    const DetectionSet& detections,
+    std::vector<SalientRegion>* required_regions,
+    std::vector<SalientRegion>* non_required_regions) {
+  required_regions->clear();
+  non_required_regions->clear();
+
+  // Makes pairs of score and index.
+  std::vector<std::pair<float, int>> required_score_idx_pairs;
+  std::vector<std::pair<float, int>> non_required_score_idx_pairs;
+  for (int i = 0; i < detections.detections_size(); ++i) {
+    const auto& detection = detections.detections(i);
+    const auto pair = std::make_pair(detection.score(), i);
+    if (detection.is_required()) {
+      required_score_idx_pairs.push_back(pair);
+    } else {
+      non_required_score_idx_pairs.push_back(pair);
+    }
+  }
+
+  // Sorts required regions by score.
+  std::stable_sort(required_score_idx_pairs.begin(),
+                   required_score_idx_pairs.end(), PairCompare);
+  for (int i = 0; i < required_score_idx_pairs.size(); ++i) {
+    const int original_idx = required_score_idx_pairs[i].second;
+    required_regions->push_back(detections.detections(original_idx));
+  }
+
+  // Sorts non-required regions by score.
+  std::stable_sort(non_required_score_idx_pairs.begin(),
+                   non_required_score_idx_pairs.end(), PairCompare);
+  for (int i = 0; i < non_required_score_idx_pairs.size(); ++i) {
+    const int original_idx = non_required_score_idx_pairs[i].second;
+    non_required_regions->push_back(detections.detections(original_idx));
+  }
+
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status SetKeyFrameCropTarget(const int frame_width,
+                                          const int frame_height,
+                                          const double target_aspect_ratio,
+                                          KeyFrameCropOptions* crop_options) {
+  RET_CHECK_NE(crop_options, nullptr) << "KeyFrameCropOptions is null.";
+  RET_CHECK_GT(frame_width, 0) << "Frame width is non-positive.";
+  RET_CHECK_GT(frame_height, 0) << "Frame height is non-positive.";
+  RET_CHECK_GT(target_aspect_ratio, 0)
+      << "Target aspect ratio is non-positive.";
+  const double input_aspect_ratio =
+      static_cast<double>(frame_width) / frame_height;
+  const int crop_target_width =
+      target_aspect_ratio < input_aspect_ratio
+          ? std::round(frame_height * target_aspect_ratio)
+          : frame_width;
+  const int crop_target_height =
+      target_aspect_ratio < input_aspect_ratio
+          ? frame_height
+          : std::round(frame_width / target_aspect_ratio);
+  crop_options->set_target_width(crop_target_width);
+  crop_options->set_target_height(crop_target_height);
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status AggregateKeyFrameResults(
+    const std::vector<KeyFrameInfo>& key_frame_infos,
+    const KeyFrameCropOptions& key_frame_crop_options,
+    const std::vector<KeyFrameCropResult>& key_frame_crop_results,
+    const int scene_frame_width, const int scene_frame_height,
+    SceneKeyFrameCropSummary* scene_summary) {
+  RET_CHECK_NE(scene_summary, nullptr)
+      << "Output SceneKeyFrameCropSummary is null.";
+
+  const int num_key_frames = key_frame_infos.size();
+  RET_CHECK_EQ(num_key_frames, key_frame_crop_results.size())
+      << "Inconsistent number of key frames:"
+      << " num_key_frames = " << num_key_frames
+      << " key_frame_crop_results.size() = " << key_frame_crop_results.size();
+
+  RET_CHECK_GT(scene_frame_width, 0) << "Non-positive frame width.";
+  RET_CHECK_GT(scene_frame_height, 0) << "Non-positive frame height.";
+
+  const int target_width = key_frame_crop_options.target_width();
+  const int target_height = key_frame_crop_options.target_height();
+  RET_CHECK_GT(target_width, 0) << "Non-positive target width.";
+  RET_CHECK_GT(target_height, 0) << "Non-positive target height.";
+  RET_CHECK_LE(target_width, scene_frame_width)
+      << "Target width exceeds frame width.";
+  RET_CHECK_LE(target_height, scene_frame_height)
+      << "Target height exceeds frame height.";
+
+  scene_summary->set_scene_frame_width(scene_frame_width);
+  scene_summary->set_scene_frame_height(scene_frame_height);
+  scene_summary->set_crop_window_width(target_width);
+  scene_summary->set_crop_window_height(target_height);
+
+  // Handles the corner case of no key frames.
+  if (num_key_frames == 0) {
+    scene_summary->set_has_salient_region(false);
+    return ::mediapipe::OkStatus();
+  }
+
+  scene_summary->set_num_key_frames(num_key_frames);
+  scene_summary->set_key_frame_center_min_x(scene_frame_width);
+  scene_summary->set_key_frame_center_max_x(0);
+  scene_summary->set_key_frame_center_min_y(scene_frame_height);
+  scene_summary->set_key_frame_center_max_y(0);
+  scene_summary->set_key_frame_min_score(std::numeric_limits<float>::max());
+  scene_summary->set_key_frame_max_score(0.0);
+
+  const float half_height = target_height / 2.0f;
+  const float half_width = target_width / 2.0f;
+  bool has_salient_region = false;
+  int num_success_frames = 0;
+  std::unique_ptr<Rect> required_crop_region_union = nullptr;
+  for (int i = 0; i < num_key_frames; ++i) {
+    auto* key_frame_compact_info = scene_summary->add_key_frame_compact_infos();
+    key_frame_compact_info->set_timestamp_ms(key_frame_infos[i].timestamp_ms());
+    const auto& result = key_frame_crop_results[i];
+    if (result.are_required_regions_covered_in_target_size()) {
+      num_success_frames++;
+    }
+    if (result.region_is_empty()) {
+      key_frame_compact_info->set_center_x(-1.0);
+      key_frame_compact_info->set_center_y(-1.0);
+      key_frame_compact_info->set_score(-1.0);
+      continue;
+    }
+
+    has_salient_region = true;
+    if (!result.required_region_is_empty()) {
+      if (required_crop_region_union == nullptr) {
+        required_crop_region_union =
+            absl::make_unique<Rect>(result.required_region());
+      } else {
+        RectUnion(result.required_region(), required_crop_region_union.get());
+      }
+    }
+
+    const auto& region = result.region();
+    float original_center_x = region.x() + region.width() / 2.0f;
+    float original_center_y = region.y() + region.height() / 2.0f;
+    RET_CHECK_GE(original_center_x, 0) << "Negative horizontal center.";
+    RET_CHECK_GE(original_center_y, 0) << "Negative vertical center.";
+    // Ensure that centered region of target size does not exceed frame size.
+    float center_x, center_y;
+    RET_CHECK(MathUtil::Clamp(half_width, scene_frame_width - half_width,
+                              original_center_x, &center_x));
+    RET_CHECK(MathUtil::Clamp(half_height, scene_frame_height - half_height,
+                              original_center_y, &center_y));
+    key_frame_compact_info->set_center_x(center_x);
+    key_frame_compact_info->set_center_y(center_y);
+    scene_summary->set_key_frame_center_min_x(
+        std::min(scene_summary->key_frame_center_min_x(), center_x));
+    scene_summary->set_key_frame_center_max_x(
+        std::max(scene_summary->key_frame_center_max_x(), center_x));
+    scene_summary->set_key_frame_center_min_y(
+        std::min(scene_summary->key_frame_center_min_y(), center_y));
+    scene_summary->set_key_frame_center_max_y(
+        std::max(scene_summary->key_frame_center_max_y(), center_y));
+
+    scene_summary->set_crop_window_width(
+        std::max(scene_summary->crop_window_width(), region.width()));
+    scene_summary->set_crop_window_height(
+        std::max(scene_summary->crop_window_height(), region.height()));
+
+    const float score = result.region_score();
+    RET_CHECK_GE(score, 0.0) << "Negative score.";
+    key_frame_compact_info->set_score(result.region_score());
+    scene_summary->set_key_frame_min_score(
+        std::min(scene_summary->key_frame_min_score(), score));
+    scene_summary->set_key_frame_max_score(
+        std::max(scene_summary->key_frame_max_score(), score));
+  }
+
+  scene_summary->set_has_salient_region(has_salient_region);
+  scene_summary->set_has_required_salient_region(required_crop_region_union !=
+                                                 nullptr);
+  if (required_crop_region_union) {
+    *(scene_summary->mutable_key_frame_required_crop_region_union()) =
+        *required_crop_region_union;
+  }
+  const float success_rate =
+      static_cast<float>(num_success_frames) / num_key_frames;
+  scene_summary->set_frame_success_rate(success_rate);
+  const float motion_x =
+      static_cast<float>(scene_summary->key_frame_center_max_x() -
+                         scene_summary->key_frame_center_min_x()) /
+      scene_frame_width;
+  scene_summary->set_horizontal_motion_amount(motion_x);
+  const float motion_y =
+      static_cast<float>(scene_summary->key_frame_center_max_y() -
+                         scene_summary->key_frame_center_min_y()) /
+      scene_frame_height;
+  scene_summary->set_vertical_motion_amount(motion_y);
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status ComputeSceneStaticBordersSize(
+    const std::vector<StaticFeatures>& static_features, int* top_border_size,
+    int* bottom_border_size) {
+  RET_CHECK(top_border_size) << "Output top border size is null.";
+  RET_CHECK(bottom_border_size) << "Output bottom border size is null.";
+
+  *top_border_size = -1;
+  for (int i = 0; i < static_features.size(); ++i) {
+    bool has_static_top_border = false;
+    for (const auto& feature : static_features[i].border()) {
+      if (feature.relative_position() == Border::TOP) {
+        has_static_top_border = true;
+        const int static_size = feature.border_position().height();
+        *top_border_size = (*top_border_size > 0)
+                               ? std::min(*top_border_size, static_size)
+                               : static_size;
+      }
+    }
+    if (!has_static_top_border) {
+      *top_border_size = 0;
+      break;
+    }
+  }
+
+  *bottom_border_size = -1;
+  for (int i = 0; i < static_features.size(); ++i) {
+    bool has_static_bottom_border = false;
+    for (const auto& feature : static_features[i].border()) {
+      if (feature.relative_position() == Border::BOTTOM) {
+        has_static_bottom_border = true;
+        const int static_size = feature.border_position().height();
+        *bottom_border_size = (*bottom_border_size > 0)
+                                  ? std::min(*bottom_border_size, static_size)
+                                  : static_size;
+      }
+    }
+    if (!has_static_bottom_border) {
+      *bottom_border_size = 0;
+      break;
+    }
+  }
+
+  *top_border_size = std::max(0, *top_border_size);
+  *bottom_border_size = std::max(0, *bottom_border_size);
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status FindSolidBackgroundColor(
+    const std::vector<StaticFeatures>& static_features,
+    const std::vector<int64>& static_features_timestamps,
+    const double min_fraction_solid_background_color,
+    bool* has_solid_background,
+    PiecewiseLinearFunction* background_color_l_function,
+    PiecewiseLinearFunction* background_color_a_function,
+    PiecewiseLinearFunction* background_color_b_function) {
+  RET_CHECK(has_solid_background) << "Output boolean is null.";
+  RET_CHECK(background_color_l_function) << "Output color l function is null.";
+  RET_CHECK(background_color_a_function) << "Output color a function is null.";
+  RET_CHECK(background_color_b_function) << "Output color b function is null.";
+
+  *has_solid_background = false;
+  int solid_background_frames = 0;
+  for (int i = 0; i < static_features.size(); ++i) {
+    if (static_features[i].has_solid_background()) {
+      solid_background_frames++;
+      const auto& color = static_features[i].solid_background();
+      const int64 timestamp = static_features_timestamps[i];
+      // BorderDetectionCalculator sets color assuming the input frame is
+      // BGR, but in reality we have RGB, so we need to revert it here.
+      // TODO remove this custom logic in BorderDetectionCalculator,
+      // original CroppingCalculator, and this calculator.
+      cv::Mat3f rgb_mat(1, 1, cv::Vec3b(color.b(), color.g(), color.r()));
+      // Necessary scaling of the RGB values from [0, 255] to [0, 1] based on:
+      // https://docs.opencv.org/2.4/modules/imgproc/doc/miscellaneous_transformations.html#cvtcolor
+      rgb_mat *= 1.0 / 255;
+      cv::Mat3f lab_mat(1, 1);
+      cv::cvtColor(rgb_mat, lab_mat, cv::COLOR_RGB2Lab);
+      // TODO change to piecewise constant interpolation if there is
+      // visual artifact. We can simply add one more point right before the
+      // next point with same value to mimic piecewise constant behavior.
+      const auto lab = lab_mat.at<cv::Vec3f>(0, 0);
+      background_color_l_function->AddPoint(timestamp, lab[0]);
+      background_color_a_function->AddPoint(timestamp, lab[1]);
+      background_color_b_function->AddPoint(timestamp, lab[2]);
+    }
+  }
+
+  if (!static_features.empty() &&
+      static_cast<float>(solid_background_frames) / static_features.size() >=
+          min_fraction_solid_background_color) {
+    *has_solid_background = true;
+  }
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status AffineRetarget(
+    const cv::Size& output_size, const std::vector<cv::Mat>& frames,
+    const std::vector<cv::Mat>& affine_projection,
+    std::vector<cv::Mat>* cropped_frames) {
+  RET_CHECK(frames.size() == affine_projection.size())
+      << "number of frames and retarget offsets must be the same.";
+  RET_CHECK(cropped_frames->size() == frames.size())
+      << "Output vector cropped_frames must be populated with output images of "
+         "the same type, size and count.";
+  for (int i = 0; i < frames.size(); i++) {
+    RET_CHECK(frames[i].type() == (*cropped_frames)[i].type())
+        << "input and output images must be the same type.";
+    const auto affine = affine_projection[i];
+    RET_CHECK(affine.cols == 3) << "Affine matrix must be 2x3";
+    RET_CHECK(affine.rows == 2) << "Affine matrix must be 2x3";
+    cv::warpAffine(frames[i], (*cropped_frames)[i], affine, output_size);
+  }
+  return ::mediapipe::OkStatus();
+}
+}  // namespace autoflip
+}  // namespace mediapipe

mediapipe/examples/desktop/autoflip/quality/utils.h

@@ -0,0 +1,119 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_UTILS_H_
+#define MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_UTILS_H_
+
+#include <vector>
+
+#include "mediapipe/examples/desktop/autoflip/autoflip_messages.pb.h"
+#include "mediapipe/examples/desktop/autoflip/quality/cropping.pb.h"
+#include "mediapipe/examples/desktop/autoflip/quality/piecewise_linear_function.h"
+#include "mediapipe/framework/port/opencv_core_inc.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+namespace autoflip {
+
+// Packs detected features and timestamp (ms) into a KeyFrameInfo object. Scales
+// features back to the original frame size if features have been detected on a
+// different frame size.
+::mediapipe::Status PackKeyFrameInfo(const int64 frame_timestamp_ms,
+                                     const DetectionSet& detections,
+                                     const int original_frame_width,
+                                     const int original_frame_height,
+                                     const int feature_frame_width,
+                                     const int feature_frame_height,
+                                     KeyFrameInfo* key_frame_info);
+
+// Sorts required and non-required salient regions given a detection set.
+::mediapipe::Status SortDetections(
+    const DetectionSet& detections,
+    std::vector<SalientRegion>* required_regions,
+    std::vector<SalientRegion>* non_required_regions);
+
+// Sets the target crop size in KeyFrameCropOptions based on frame size and
+// target aspect ratio so that the target crop size covers the biggest area
+// possible in the frame.
+::mediapipe::Status SetKeyFrameCropTarget(const int frame_width,
+                                          const int frame_height,
+                                          const double target_aspect_ratio,
+                                          KeyFrameCropOptions* crop_options);
+
+// Aggregates information from KeyFrameInfos and KeyFrameCropResults into
+// SceneKeyFrameCropSummary.
+::mediapipe::Status AggregateKeyFrameResults(
+    const std::vector<KeyFrameInfo>& key_frame_infos,
+    const KeyFrameCropOptions& key_frame_crop_options,
+    const std::vector<KeyFrameCropResult>& key_frame_crop_results,
+    const int scene_frame_width, const int scene_frame_height,
+    SceneKeyFrameCropSummary* scene_summary);
+
+// Computes the static top and border size across a scene given a vector of
+// StaticFeatures over frames.
+::mediapipe::Status ComputeSceneStaticBordersSize(
+    const std::vector<StaticFeatures>& static_features, int* top_border_size,
+    int* bottom_border_size);
+
+// Finds the solid background colors in a scene from input StaticFeatures.
+// Sets has_solid_background to true if the number of frames with solid
+// background color exceeds given threshold, i.e.,
+// min_fraction_solid_background_color. Builds the background color
+// interpolation functions in Lab space using input timestamps.
+::mediapipe::Status FindSolidBackgroundColor(
+    const std::vector<StaticFeatures>& static_features,
+    const std::vector<int64>& static_features_timestamps,
+    const double min_fraction_solid_background_color,
+    bool* has_solid_background,
+    PiecewiseLinearFunction* background_color_l_function,
+    PiecewiseLinearFunction* background_color_a_function,
+    PiecewiseLinearFunction* background_color_b_function);
+
+// Helpers to scale, clamp, and take union of rectangles. These functions do not
+// check for pointers not being null or rectangles being valid.
+
+// Scales a rectangle given horizontal and vertical scaling factors.
+template <typename T>
+void ScaleRect(const T& original_location, const double scale_x,
+               const double scale_y, Rect* scaled_location);
+
+// Converts a normalized rectangle to a rectangle given width and height.
+void NormalizedRectToRect(const RectF& normalized_location, const int width,
+                          const int height, Rect* location);
+
+// Clamps a rectangle to lie within [x0, y0] and [x1, y1]. Returns true if the
+// rectangle has any overlapping with the target window.
+::mediapipe::Status ClampRect(const int x0, const int y0, const int x1,
+                              const int y1, Rect* location);
+
+// Convenience function to clamp a rectangle to lie within [0, 0] and
+// [width, height].
+::mediapipe::Status ClampRect(const int width, const int height,
+                              Rect* location);
+
+// Enlarges a given rectangle to cover a new rectangle to be added.
+void RectUnion(const Rect& rect_to_add, Rect* rect);
+
+// Performs an affine retarget on a list of input images.  Output vector
+// cropped_frames must be filled with Mats of the same size as output_size and
+// type.
+::mediapipe::Status AffineRetarget(
+    const cv::Size& output_size, const std::vector<cv::Mat>& frames,
+    const std::vector<cv::Mat>& affine_projection,
+    std::vector<cv::Mat>* cropped_frames);
+
+}  // namespace autoflip
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_UTILS_H_

mediapipe/examples/desktop/autoflip/quality/visual_scorer.cc

@@ -0,0 +1,182 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/examples/desktop/autoflip/quality/visual_scorer.h"
+
+#include <math.h>
+
+#include <algorithm>
+#include <cmath>
+#include <memory>
+#include <vector>
+
+#include "mediapipe/framework/port/opencv_core_inc.h"
+#include "mediapipe/framework/port/opencv_imgproc_inc.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/port/status_builder.h"
+
+// Weight threshold for computing a value.
+constexpr float kEpsilon = 0.0001;
+
+namespace mediapipe {
+namespace autoflip {
+namespace {
+
+// Crop the given rectangle so that it fits in the given 2D matrix.
+void CropRectToMat(const cv::Mat& image, cv::Rect* rect) {
+  int x = std::min(std::max(rect->x, 0), image.cols);
+  int y = std::min(std::max(rect->y, 0), image.rows);
+  int w = std::min(std::max(rect->x + rect->width, 0), image.cols) - x;
+  int h = std::min(std::max(rect->y + rect->height, 0), image.rows) - y;
+  *rect = cv::Rect(x, y, w, h);
+}
+
+}  // namespace
+
+VisualScorer::VisualScorer(const VisualScorerOptions& options)
+    : options_(options) {}
+
+mediapipe::Status VisualScorer::CalculateScore(const cv::Mat& image,
+                                               const SalientRegion& region,
+                                               float* score) const {
+  const float weight_sum = options_.area_weight() +
+                           options_.sharpness_weight() +
+                           options_.colorfulness_weight();
+
+  // Crop the region to fit in the image.
+  cv::Rect region_rect;
+  if (region.has_location()) {
+    region_rect =
+        cv::Rect(region.location().x(), region.location().y(),
+                 region.location().width(), region.location().height());
+  } else if (region.has_location_normalized()) {
+    region_rect = cv::Rect(region.location_normalized().x() * image.cols,
+                           region.location_normalized().y() * image.rows,
+                           region.location_normalized().width() * image.cols,
+                           region.location_normalized().height() * image.rows);
+  } else {
+    return ::mediapipe::UnknownErrorBuilder(MEDIAPIPE_LOC)
+           << "Unset region location.";
+  }
+
+  CropRectToMat(image, &region_rect);
+  if (region_rect.area() == 0) {
+    *score = 0;
+    return ::mediapipe::OkStatus();
+  }
+
+  // Compute a score based on area covered by this region.
+  const float area_score =
+      options_.area_weight() * region_rect.area() / (image.cols * image.rows);
+
+  // Convert the visible region to cv::Mat.
+  cv::Mat image_region_mat = image(region_rect);
+
+  // Compute a score from sharpness.
+
+  float sharpness_score_result = 0.0;
+  if (options_.sharpness_weight() > kEpsilon) {
+    // TODO: implement a sharpness score or remove this code block.
+    return ::mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
+           << "sharpness scorer is not yet implemented, please set weight to "
+              "0.0";
+  }
+  const float sharpness_score =
+      options_.sharpness_weight() * sharpness_score_result;
+
+  // Compute a colorfulness score.
+  float colorfulness_score = 0;
+  if (options_.colorfulness_weight() > kEpsilon) {
+    MP_RETURN_IF_ERROR(
+        CalculateColorfulness(image_region_mat, &colorfulness_score));
+    colorfulness_score *= options_.colorfulness_weight();
+  }
+
+  *score = (area_score + sharpness_score + colorfulness_score) / weight_sum;
+  if (*score > 1.0f || *score < 0.0f) {
+    LOG(WARNING) << "Score of region outside expected range: " << *score;
+  }
+  return ::mediapipe::OkStatus();
+}
+
+mediapipe::Status VisualScorer::CalculateColorfulness(
+    const cv::Mat& image, float* colorfulness) const {
+  // Convert the image to HSV.
+  cv::Mat image_hsv;
+  cv::cvtColor(image, image_hsv, CV_RGB2HSV);
+
+  // Mask out pixels that are too dark or too bright.
+  cv::Mat mask(image.rows, image.cols, CV_8UC1);
+  bool empty_mask = true;
+  for (int x = 0; x < image.cols; ++x) {
+    for (int y = 0; y < image.rows; ++y) {
+      const cv::Vec3b& pixel = image.at<cv::Vec3b>(x, y);
+      const bool is_usable =
+          (std::min(pixel.val[0], std::min(pixel.val[1], pixel.val[2])) < 250 &&
+           std::max(pixel.val[0], std::max(pixel.val[1], pixel.val[2])) > 5);
+      mask.at<unsigned char>(y, x) = is_usable ? 255 : 0;
+      if (is_usable) empty_mask = false;
+    }
+  }
+
+  // If the mask is empty, return.
+  if (empty_mask) {
+    *colorfulness = 0;
+    return ::mediapipe::OkStatus();
+  }
+
+  // Generate a 2D histogram (hue/saturation).
+  cv::MatND hs_histogram;
+  const int kHueBins = 10, kSaturationBins = 8;
+  const int kHistogramChannels[] = {0, 1};
+  const int kHistogramBinNum[] = {kHueBins, kSaturationBins};
+  const float kHueRange[] = {0, 180};
+  const float kSaturationRange[] = {0, 256};
+  const float* kHistogramRange[] = {kHueRange, kSaturationRange};
+  cv::calcHist(&image_hsv, 1, kHistogramChannels, mask, hs_histogram,
+               2 /* histogram dims */, kHistogramBinNum, kHistogramRange,
+               true /* uniform */, false /* accumulate */);
+
+  // Convert to a hue histogram and weigh saturated pixels more.
+  std::vector<float> hue_histogram(kHueBins, 0.0f);
+  float hue_sum = 0.0f;
+  for (int bin_s = 0; bin_s < kSaturationBins; ++bin_s) {
+    const float weight = std::pow(2.0f, bin_s);
+    for (int bin_h = 0; bin_h < kHueBins; ++bin_h) {
+      float value = hs_histogram.at<float>(bin_h, bin_s) * weight;
+      hue_histogram[bin_h] += value;
+      hue_sum += value;
+    }
+  }
+  if (hue_sum == 0.0f) {
+    *colorfulness = 0;
+    return ::mediapipe::OkStatus();
+  }
+
+  // Compute the histogram entropy.
+  *colorfulness = 0;
+  for (int bin = 0; bin < kHueBins; ++bin) {
+    float value = hue_histogram[bin] / hue_sum;
+    if (value > 0.0f) {
+      *colorfulness -= value * std::log(value);
+    }
+  }
+  *colorfulness /= std::log(2.0f);
+
+  return ::mediapipe::OkStatus();
+}
+
+}  // namespace autoflip
+}  // namespace mediapipe

mediapipe/examples/desktop/autoflip/quality/visual_scorer.h

@@ -0,0 +1,47 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_VISUAL_SCORER_H_
+#define MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_VISUAL_SCORER_H_
+
+#include "mediapipe/examples/desktop/autoflip/autoflip_messages.pb.h"
+#include "mediapipe/examples/desktop/autoflip/quality/visual_scorer.pb.h"
+#include "mediapipe/framework/port/opencv_core_inc.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+namespace autoflip {
+
+// This class scores a SalientRegion within an image based on weighted averages
+// of various signals computed on the patch.
+class VisualScorer {
+ public:
+  explicit VisualScorer(const VisualScorerOptions& options);
+
+  // Computes a score on a salientregion and returns a value [0...1].
+  mediapipe::Status CalculateScore(const cv::Mat& image,
+                                   const SalientRegion& region,
+                                   float* score) const;
+
+ private:
+  mediapipe::Status CalculateColorfulness(const cv::Mat& image,
+                                          float* colorfulness) const;
+
+  VisualScorerOptions options_;
+};
+
+}  // namespace autoflip
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_VISUAL_SCORER_H_

mediapipe/examples/desktop/autoflip/quality/visual_scorer.proto

@@ -0,0 +1,28 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe.autoflip;
+
+// Options for the VisualScorer module.
+// Next tag: 6
+message VisualScorerOptions {
+  // Weights for the various cues. A larger weight means that the corresponding
+  // cue will be of higher importance when generating the combined score.
+  optional float area_weight = 1 [default = 1.0];
+  // Sharpness is not yet implemented.
+  optional float sharpness_weight = 2 [default = 0.0];
+  optional float colorfulness_weight = 3 [default = 0.0];
+}

mediapipe/examples/desktop/autoflip/quality/visual_scorer_test.cc

@@ -0,0 +1,80 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/examples/desktop/autoflip/quality/visual_scorer.h"
+
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/opencv_core_inc.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/status_matchers.h"
+
+namespace mediapipe {
+namespace autoflip {
+namespace {
+
+TEST(VisualScorerTest, ScoresArea) {
+  cv::Mat image_mat(200, 200, CV_8UC3);
+  SalientRegion region = ParseTextProtoOrDie<SalientRegion>(
+      R"(location { x: 10 y: 10 width: 100 height: 100 })");
+
+  VisualScorerOptions options = ParseTextProtoOrDie<VisualScorerOptions>(
+      R"(area_weight: 1.0 sharpness_weight: 0 colorfulness_weight: 0)");
+  VisualScorer scorer(options);
+  float score = 0.0;
+  MP_EXPECT_OK(scorer.CalculateScore(image_mat, region, &score));
+  EXPECT_EQ(0.25, score);  // (100 * 100) / (200 * 200).
+}
+
+TEST(VisualScorerTest, ScoresSharpness) {
+  SalientRegion region = ParseTextProtoOrDie<SalientRegion>(
+      R"(location { x: 10 y: 10 width: 100 height: 100 })");
+
+  VisualScorerOptions options = ParseTextProtoOrDie<VisualScorerOptions>(
+      R"(area_weight: 0 sharpness_weight: 1.0 colorfulness_weight: 0)");
+  VisualScorer scorer(options);
+
+  // Compute the score of an empty image and an image with a rectangle.
+  cv::Mat image_mat(200, 200, CV_8UC3);
+  image_mat.setTo(cv::Scalar(0, 0, 0));
+  float score_rect = 0;
+  auto status = scorer.CalculateScore(image_mat, region, &score_rect);
+  EXPECT_EQ(status.code(), StatusCode::kInvalidArgument);
+}
+
+TEST(VisualScorerTest, ScoresColorfulness) {
+  SalientRegion region = ParseTextProtoOrDie<SalientRegion>(
+      R"(location { x: 10 y: 10 width: 50 height: 150 })");
+
+  VisualScorerOptions options = ParseTextProtoOrDie<VisualScorerOptions>(
+      R"(area_weight: 0 sharpness_weight: 0 colorfulness_weight: 1.0)");
+  VisualScorer scorer(options);
+
+  // Compute the scores of images with 1, 2 and 3 colors.
+  cv::Mat image_mat(200, 200, CV_8UC3);
+  image_mat.setTo(cv::Scalar(0, 0, 255));
+  float score_1c = 0, score_2c = 0, score_3c = 0;
+  MP_EXPECT_OK(scorer.CalculateScore(image_mat, region, &score_1c));
+  image_mat(cv::Rect(30, 30, 20, 20)).setTo(cv::Scalar(128, 0, 0));
+  MP_EXPECT_OK(scorer.CalculateScore(image_mat, region, &score_2c));
+  image_mat(cv::Rect(50, 50, 20, 20)).setTo(cv::Scalar(255, 128, 0));
+  MP_EXPECT_OK(scorer.CalculateScore(image_mat, region, &score_3c));
+  // Images with more colors should have a higher score.
+  EXPECT_LT(score_1c, score_2c);
+  EXPECT_LT(score_2c, score_3c);
+}
+
+}  // namespace
+}  // namespace autoflip
+}  // namespace mediapipe

mediapipe/examples/desktop/autoflip/subgraph/BUILD

@@ -0,0 +1,39 @@
+load("//mediapipe/framework/tool:mediapipe_graph.bzl", "mediapipe_simple_subgraph")
+
+# Copyright 2019 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+licenses(["notice"])  # Apache 2.0
+
+package(default_visibility = ["//mediapipe/examples:__subpackages__"])
+
+mediapipe_simple_subgraph(
+    name = "autoflip_face_detection_subgraph",
+    graph = "face_detection_subgraph.pbtxt",
+    register_as = "AutoFlipFaceDetectionSubgraph",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/graphs/face_detection:desktop_tflite_calculators",
+    ],
+)
+
+mediapipe_simple_subgraph(
+    name = "autoflip_object_detection_subgraph",
+    graph = "autoflip_object_detection_subgraph.pbtxt",
+    register_as = "AutoFlipObjectDetectionSubgraph",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/graphs/object_detection:desktop_tflite_calculators",
+    ],
+)

mediapipe/examples/desktop/autoflip/subgraph/autoflip_object_detection_subgraph.pbtxt

@@ -0,0 +1,126 @@
+# MediaPipe graph that performs object detection with TensorFlow Lite on CPU.
+
+input_stream: "VIDEO:input_video"
+output_stream: "DETECTIONS:output_detections"
+
+# Transforms the input image on CPU to a 320x320 image. To scale the image, by
+# default it uses the STRETCH scale mode that maps the entire input image to the
+# entire transformed image. As a result, image aspect ratio may be changed and
+# objects in the image may be deformed (stretched or squeezed), but the object
+# detection model used in this graph is agnostic to that deformation.
+node: {
+  calculator: "ImageTransformationCalculator"
+  input_stream: "IMAGE:input_video"
+  output_stream: "IMAGE:transformed_input_video"
+  options: {
+    [mediapipe.ImageTransformationCalculatorOptions.ext] {
+      output_width: 320
+      output_height: 320
+    }
+  }
+}
+
+# Converts the transformed input image on CPU into an image tensor stored as a
+# TfLiteTensor.
+node {
+  calculator: "TfLiteConverterCalculator"
+  input_stream: "IMAGE:transformed_input_video"
+  output_stream: "TENSORS:image_tensor"
+}
+
+# Runs a TensorFlow Lite model on CPU that takes an image tensor and outputs a
+# vector of tensors representing, for instance, detection boxes/keypoints and
+# scores.
+node {
+  calculator: "TfLiteInferenceCalculator"
+  input_stream: "TENSORS:image_tensor"
+  output_stream: "TENSORS:detection_tensors"
+  options: {
+    [mediapipe.TfLiteInferenceCalculatorOptions.ext] {
+      model_path: "mediapipe/models/ssdlite_object_detection.tflite"
+    }
+  }
+}
+
+# Generates a single side packet containing a vector of SSD anchors based on
+# the specification in the options.
+node {
+  calculator: "SsdAnchorsCalculator"
+  output_side_packet: "anchors"
+  options: {
+    [mediapipe.SsdAnchorsCalculatorOptions.ext] {
+      num_layers: 6
+      min_scale: 0.2
+      max_scale: 0.95
+      input_size_height: 320
+      input_size_width: 320
+      anchor_offset_x: 0.5
+      anchor_offset_y: 0.5
+      strides: 16
+      strides: 32
+      strides: 64
+      strides: 128
+      strides: 256
+      strides: 512
+      aspect_ratios: 1.0
+      aspect_ratios: 2.0
+      aspect_ratios: 0.5
+      aspect_ratios: 3.0
+      aspect_ratios: 0.3333
+      reduce_boxes_in_lowest_layer: true
+    }
+  }
+}
+
+# Decodes the detection tensors generated by the TensorFlow Lite model, based on
+# the SSD anchors and the specification in the options, into a vector of
+# detections. Each detection describes a detected object.
+node {
+  calculator: "TfLiteTensorsToDetectionsCalculator"
+  input_stream: "TENSORS:detection_tensors"
+  input_side_packet: "ANCHORS:anchors"
+  output_stream: "DETECTIONS:detections"
+  options: {
+    [mediapipe.TfLiteTensorsToDetectionsCalculatorOptions.ext] {
+      num_classes: 91
+      num_boxes: 2034
+      num_coords: 4
+      ignore_classes: 0
+      sigmoid_score: true
+      apply_exponential_on_box_size: true
+      x_scale: 10.0
+      y_scale: 10.0
+      h_scale: 5.0
+      w_scale: 5.0
+      min_score_thresh: 0.6
+    }
+  }
+}
+
+# Performs non-max suppression to remove excessive detections.
+node {
+  calculator: "NonMaxSuppressionCalculator"
+  input_stream: "detections"
+  output_stream: "filtered_detections"
+  options: {
+    [mediapipe.NonMaxSuppressionCalculatorOptions.ext] {
+      min_suppression_threshold: 0.4
+      max_num_detections: 5
+      overlap_type: INTERSECTION_OVER_UNION
+      return_empty_detections: true
+    }
+  }
+}
+
+# Maps detection label IDs to the corresponding label text. The label map is
+# provided in the label_map_path option.
+node {
+  calculator: "DetectionLabelIdToTextCalculator"
+  input_stream: "filtered_detections"
+  output_stream: "output_detections"
+  options: {
+    [mediapipe.DetectionLabelIdToTextCalculatorOptions.ext] {
+      label_map_path: "mediapipe/models/ssdlite_object_detection_labelmap.txt"
+    }
+  }
+}

mediapipe/examples/desktop/autoflip/subgraph/face_detection_subgraph.pbtxt

@@ -0,0 +1,136 @@
+# MediaPipe graph that performs face detection with TensorFlow Lite on CPU.
+
+input_stream: "VIDEO:input_video"
+output_stream: "DETECTIONS:output_detections"
+
+
+# Transforms the input image on CPU to a 128x128 image. To scale the input
+# image, the scale_mode option is set to FIT to preserve the aspect ratio,
+# resulting in potential letterboxing in the transformed image.
+node: {
+  calculator: "ImageTransformationCalculator"
+  input_stream: "IMAGE:input_video"
+  output_stream: "IMAGE:transformed_input_video_cpu"
+  output_stream: "LETTERBOX_PADDING:letterbox_padding"
+  options: {
+    [mediapipe.ImageTransformationCalculatorOptions.ext] {
+      output_width: 128
+      output_height: 128
+      scale_mode: FIT
+    }
+  }
+}
+
+# Converts the transformed input image on CPU into an image tensor stored as a
+# TfLiteTensor.
+node {
+  calculator: "TfLiteConverterCalculator"
+  input_stream: "IMAGE:transformed_input_video_cpu"
+  output_stream: "TENSORS:image_tensor"
+}
+
+# Runs a TensorFlow Lite model on CPU that takes an image tensor and outputs a
+# vector of tensors representing, for instance, detection boxes/keypoints and
+# scores.
+node {
+  calculator: "TfLiteInferenceCalculator"
+  input_stream: "TENSORS:image_tensor"
+  output_stream: "TENSORS:detection_tensors"
+  options: {
+    [mediapipe.TfLiteInferenceCalculatorOptions.ext] {
+      model_path: "mediapipe/models/face_detection_front.tflite"
+    }
+  }
+}
+
+# Generates a single side packet containing a vector of SSD anchors based on
+# the specification in the options.
+node {
+  calculator: "SsdAnchorsCalculator"
+  output_side_packet: "anchors"
+  options: {
+    [mediapipe.SsdAnchorsCalculatorOptions.ext] {
+      num_layers: 4
+      min_scale: 0.1484375
+      max_scale: 0.75
+      input_size_height: 128
+      input_size_width: 128
+      anchor_offset_x: 0.5
+      anchor_offset_y: 0.5
+      strides: 8
+      strides: 16
+      strides: 16
+      strides: 16
+      aspect_ratios: 1.0
+      fixed_anchor_size: true
+    }
+  }
+}
+
+# Decodes the detection tensors generated by the TensorFlow Lite model, based on
+# the SSD anchors and the specification in the options, into a vector of
+# detections. Each detection describes a detected object.
+node {
+  calculator: "TfLiteTensorsToDetectionsCalculator"
+  input_stream: "TENSORS:detection_tensors"
+  input_side_packet: "ANCHORS:anchors"
+  output_stream: "DETECTIONS:detections"
+  options: {
+    [mediapipe.TfLiteTensorsToDetectionsCalculatorOptions.ext] {
+      num_classes: 1
+      num_boxes: 896
+      num_coords: 16
+      box_coord_offset: 0
+      keypoint_coord_offset: 4
+      num_keypoints: 6
+      num_values_per_keypoint: 2
+      sigmoid_score: true
+      score_clipping_thresh: 100.0
+      reverse_output_order: true
+      x_scale: 128.0
+      y_scale: 128.0
+      h_scale: 128.0
+      w_scale: 128.0
+      min_score_thresh: 0.6
+    }
+  }
+}
+
+# Performs non-max suppression to remove excessive detections.
+node {
+  calculator: "NonMaxSuppressionCalculator"
+  input_stream: "detections"
+  output_stream: "filtered_detections"
+  options: {
+    [mediapipe.NonMaxSuppressionCalculatorOptions.ext] {
+      min_suppression_threshold: 0.3
+      overlap_type: INTERSECTION_OVER_UNION
+      algorithm: WEIGHTED
+      return_empty_detections: true
+    }
+  }
+}
+
+# Maps detection label IDs to the corresponding label text ("Face"). The label
+# map is provided in the label_map_path option.
+node {
+  calculator: "DetectionLabelIdToTextCalculator"
+  input_stream: "filtered_detections"
+  output_stream: "labeled_detections"
+  options: {
+    [mediapipe.DetectionLabelIdToTextCalculatorOptions.ext] {
+      label_map_path: "mediapipe/models/face_detection_front_labelmap.txt"
+    }
+  }
+}
+
+# Adjusts detection locations (already normalized to [0.f, 1.f]) on the
+# letterboxed image (after image transformation with the FIT scale mode) to the
+# corresponding locations on the same image with the letterbox removed (the
+# input image to the graph before image transformation).
+node {
+  calculator: "DetectionLetterboxRemovalCalculator"
+  input_stream: "DETECTIONS:labeled_detections"
+  input_stream: "LETTERBOX_PADDING:letterbox_padding"
+  output_stream: "DETECTIONS:output_detections"
+}

mediapipe/examples/desktop/demo_run_graph_main.cc

@@ -13,6 +13,7 @@
 // limitations under the License.
 //
 // An example of sending OpenCV webcam frames into a MediaPipe graph.
+#include <cstdlib>
 
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/formats/image_frame.h"
@@ -65,16 +66,7 @@ DEFINE_string(output_video_path, "",
 
   cv::VideoWriter writer;
   const bool save_video = !FLAGS_output_video_path.empty();
-  if (save_video) {
-    LOG(INFO) << "Prepare video writer.";
-    cv::Mat test_frame;
-    capture.read(test_frame);                    // Consume first frame.
-    capture.set(cv::CAP_PROP_POS_AVI_RATIO, 0);  // Rewind to beginning.
-    writer.open(FLAGS_output_video_path,
-                mediapipe::fourcc('a', 'v', 'c', '1'),  // .mp4
-                capture.get(cv::CAP_PROP_FPS), test_frame.size());
-    RET_CHECK(writer.isOpened());
-  } else {
+  if (!save_video) {
     cv::namedWindow(kWindowName, /*flags=WINDOW_AUTOSIZE*/ 1);
 #if (CV_MAJOR_VERSION >= 3) && (CV_MINOR_VERSION >= 2)
     capture.set(cv::CAP_PROP_FRAME_WIDTH, 640);
@@ -89,7 +81,6 @@ DEFINE_string(output_video_path, "",
   MP_RETURN_IF_ERROR(graph.StartRun({}));
 
   LOG(INFO) << "Start grabbing and processing frames.";
-  size_t frame_timestamp = 0;
   bool grab_frames = true;
   while (grab_frames) {
     // Capture opencv camera or video frame.
@@ -110,9 +101,11 @@ DEFINE_string(output_video_path, "",
     camera_frame.copyTo(input_frame_mat);
 
     // Send image packet into the graph.
+    size_t frame_timestamp_us =
+        (double)cv::getTickCount() / (double)cv::getTickFrequency() * 1e6;
     MP_RETURN_IF_ERROR(graph.AddPacketToInputStream(
         kInputStream, mediapipe::Adopt(input_frame.release())
-                          .At(mediapipe::Timestamp(frame_timestamp++))));
+                          .At(mediapipe::Timestamp(frame_timestamp_us))));
 
     // Get the graph result packet, or stop if that fails.
     mediapipe::Packet packet;
@@ -123,6 +116,13 @@ DEFINE_string(output_video_path, "",
     cv::Mat output_frame_mat = mediapipe::formats::MatView(&output_frame);
     cv::cvtColor(output_frame_mat, output_frame_mat, cv::COLOR_RGB2BGR);
     if (save_video) {
+      if (!writer.isOpened()) {
+        LOG(INFO) << "Prepare video writer.";
+        writer.open(FLAGS_output_video_path,
+                    mediapipe::fourcc('a', 'v', 'c', '1'),  // .mp4
+                    capture.get(cv::CAP_PROP_FPS), output_frame_mat.size());
+        RET_CHECK(writer.isOpened());
+      }
       writer.write(output_frame_mat);
     } else {
       cv::imshow(kWindowName, output_frame_mat);
@@ -144,8 +144,9 @@ int main(int argc, char** argv) {
   ::mediapipe::Status run_status = RunMPPGraph();
   if (!run_status.ok()) {
     LOG(ERROR) << "Failed to run the graph: " << run_status.message();
+    return EXIT_FAILURE;
   } else {
     LOG(INFO) << "Success!";
   }
-  return 0;
+  return EXIT_SUCCESS;
 }

mediapipe/examples/desktop/demo_run_graph_main_gpu.cc

@@ -14,6 +14,7 @@
 //
 // An example of sending OpenCV webcam frames into a MediaPipe graph.
 // This example requires a linux computer and a GPU with EGL support drivers.
+#include <cstdlib>
 
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/formats/image_frame.h"
@@ -75,16 +76,7 @@ DEFINE_string(output_video_path, "",
 
   cv::VideoWriter writer;
   const bool save_video = !FLAGS_output_video_path.empty();
-  if (save_video) {
-    LOG(INFO) << "Prepare video writer.";
-    cv::Mat test_frame;
-    capture.read(test_frame);                    // Consume first frame.
-    capture.set(cv::CAP_PROP_POS_AVI_RATIO, 0);  // Rewind to beginning.
-    writer.open(FLAGS_output_video_path,
-                mediapipe::fourcc('a', 'v', 'c', '1'),  // .mp4
-                capture.get(cv::CAP_PROP_FPS), test_frame.size());
-    RET_CHECK(writer.isOpened());
-  } else {
+  if (!save_video) {
     cv::namedWindow(kWindowName, /*flags=WINDOW_AUTOSIZE*/ 1);
 #if (CV_MAJOR_VERSION >= 3) && (CV_MINOR_VERSION >= 2)
     capture.set(cv::CAP_PROP_FRAME_WIDTH, 640);
@@ -99,7 +91,6 @@ DEFINE_string(output_video_path, "",
   MP_RETURN_IF_ERROR(graph.StartRun({}));
 
   LOG(INFO) << "Start grabbing and processing frames.";
-  size_t frame_timestamp = 0;
   bool grab_frames = true;
   while (grab_frames) {
     // Capture opencv camera or video frame.
@@ -120,8 +111,10 @@ DEFINE_string(output_video_path, "",
     camera_frame.copyTo(input_frame_mat);
 
     // Prepare and add graph input packet.
+    size_t frame_timestamp_us =
+        (double)cv::getTickCount() / (double)cv::getTickFrequency() * 1e6;
     MP_RETURN_IF_ERROR(
-        gpu_helper.RunInGlContext([&input_frame, &frame_timestamp, &graph,
+        gpu_helper.RunInGlContext([&input_frame, &frame_timestamp_us, &graph,
                                    &gpu_helper]() -> ::mediapipe::Status {
           // Convert ImageFrame to GpuBuffer.
           auto texture = gpu_helper.CreateSourceTexture(*input_frame.get());
@@ -131,7 +124,7 @@ DEFINE_string(output_video_path, "",
           // Send GPU image packet into the graph.
           MP_RETURN_IF_ERROR(graph.AddPacketToInputStream(
               kInputStream, mediapipe::Adopt(gpu_frame.release())
-                                .At(mediapipe::Timestamp(frame_timestamp++))));
+                                .At(mediapipe::Timestamp(frame_timestamp_us))));
           return ::mediapipe::OkStatus();
         }));
 
@@ -163,6 +156,13 @@ DEFINE_string(output_video_path, "",
     cv::Mat output_frame_mat = mediapipe::formats::MatView(output_frame.get());
     cv::cvtColor(output_frame_mat, output_frame_mat, cv::COLOR_RGB2BGR);
     if (save_video) {
+      if (!writer.isOpened()) {
+        LOG(INFO) << "Prepare video writer.";
+        writer.open(FLAGS_output_video_path,
+                    mediapipe::fourcc('a', 'v', 'c', '1'),  // .mp4
+                    capture.get(cv::CAP_PROP_FPS), output_frame_mat.size());
+        RET_CHECK(writer.isOpened());
+      }
       writer.write(output_frame_mat);
     } else {
       cv::imshow(kWindowName, output_frame_mat);
@@ -184,8 +184,9 @@ int main(int argc, char** argv) {
   ::mediapipe::Status run_status = RunMPPGraph();
   if (!run_status.ok()) {
     LOG(ERROR) << "Failed to run the graph: " << run_status.message();
+    return EXIT_FAILURE;
   } else {
     LOG(INFO) << "Success!";
   }
-  return 0;
+  return EXIT_SUCCESS;
 }

mediapipe/examples/desktop/media_sequence/kinetics_dataset.py

@@ -56,6 +56,9 @@ with the following lines:
 This data is structured for per-clip action classification where images is
 the sequence of images and labels are a one-hot encoded value. See
 as_dataset() for more details.
+
+Note that the number of videos changes in the data set over time, so it will
+likely be necessary to change the expected number of examples.
 """
 
 from __future__ import absolute_import
@@ -93,15 +96,15 @@ FILEPATTERN = "kinetics_700_%s_25fps_rgb_flow"
 SPLITS = {
     "train": {
         "shards": 1000,
-        "examples": 541279
+        "examples": 540247
     },
     "validate": {
         "shards": 100,
-        "examples": 34688
+        "examples": 34610
     },
     "test": {
         "shards": 100,
-        "examples": 69278
+        "examples": 69103
     },
     "custom": {
         "csv": None,  # Add a CSV for your own data here.
@@ -121,7 +124,8 @@ class Kinetics(object):
     self.path_to_data = path_to_data
 
   def as_dataset(self, split, shuffle=False, repeat=False,
-                 serialized_prefetch_size=32, decoded_prefetch_size=32):
+                 serialized_prefetch_size=32, decoded_prefetch_size=32,
+                 parse_labels=True):
     """Returns Kinetics as a tf.data.Dataset.
 
     After running this function, calling padded_batch() on the Dataset object
@@ -135,20 +139,29 @@ class Kinetics(object):
       repeat: if true, repeats the data set forever.
       serialized_prefetch_size: the buffer size for reading from disk.
       decoded_prefetch_size: the buffer size after decoding.
+      parse_labels: if true, also returns the "labels" below. The only
+        case where this should be false is if the data set was not constructed
+        with a label map, resulting in this field being missing.
     Returns:
       A tf.data.Dataset object with the following structure: {
         "images": float tensor, shape [time, height, width, channels]
         "flow": float tensor, shape [time, height, width, 2]
-        "labels": float32 tensor, shape [num_classes], one hot encoded
         "num_frames": int32 tensor, shape [], number of frames in the sequence
+        "labels": float32 tensor, shape [num_classes], one hot encoded. Only
+          present if parse_labels is true.
     """
+    logging.info("If you see an error about labels, and you don't supply "
+                 "labels in your CSV, set parse_labels=False")
     def parse_fn(sequence_example):
       """Parses a Kinetics example."""
       context_features = {
           ms.get_example_id_key(): ms.get_example_id_default_parser(),
-          ms.get_clip_label_string_key(): tf.FixedLenFeature((), tf.string),
-          ms.get_clip_label_index_key(): tf.FixedLenFeature((), tf.int64),
       }
+      if parse_labels:
+        context_features[
+            ms.get_clip_label_string_key()] = tf.FixedLenFeature((), tf.string)
+        context_features[
+            ms.get_clip_label_index_key()] = tf.FixedLenFeature((), tf.int64)
 
       sequence_features = {
           ms.get_image_encoded_key(): ms.get_image_encoded_default_parser(),
@@ -158,8 +171,6 @@ class Kinetics(object):
       parsed_context, parsed_sequence = tf.io.parse_single_sequence_example(
           sequence_example, context_features, sequence_features)
 
-      target = tf.one_hot(parsed_context[ms.get_clip_label_index_key()], 700)
-
       images = tf.image.convert_image_dtype(
           tf.map_fn(tf.image.decode_jpeg,
                     parsed_sequence[ms.get_image_encoded_key()],
@@ -177,11 +188,13 @@ class Kinetics(object):
       flow = (flow[:, :, :, :2] - 0.5) * 2 * 20.
 
       output_dict = {
-          "labels": target,
           "images": images,
           "flow": flow,
           "num_frames": num_frames,
       }
+      if parse_labels:
+        target = tf.one_hot(parsed_context[ms.get_clip_label_index_key()], 700)
+        output_dict["labels"] = target
       return output_dict
 
     if split not in SPLITS:

mediapipe/examples/desktop/media_sequence/run_graph_file_io_main.cc

@@ -15,6 +15,7 @@
 // A simple main function to run a MediaPipe graph. Input side packets are read
 // from files provided via the command line and output side packets are written
 // to disk.
+#include <cstdlib>
 
 #include "absl/strings/str_split.h"
 #include "mediapipe/framework/calculator_framework.h"
@@ -87,8 +88,9 @@ int main(int argc, char** argv) {
   ::mediapipe::Status run_status = RunMPPGraph();
   if (!run_status.ok()) {
     LOG(ERROR) << "Failed to run the graph: " << run_status.message();
+    return EXIT_FAILURE;
   } else {
     LOG(INFO) << "Success!";
   }
-  return 0;
+  return EXIT_SUCCESS;
 }

mediapipe/examples/desktop/object_tracking/BUILD

@@ -0,0 +1,25 @@
+# Copyright 2019 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+licenses(["notice"])  # Apache 2.0
+
+package(default_visibility = ["//mediapipe/examples:__subpackages__"])
+
+cc_binary(
+    name = "object_tracking_cpu",
+    deps = [
+        "//mediapipe/examples/desktop:demo_run_graph_main",
+        "//mediapipe/graphs/tracking:desktop_calculators",
+    ],
+)

mediapipe/examples/desktop/simple_run_graph_main.cc

@@ -13,6 +13,7 @@
 // limitations under the License.
 //
 // A simple main function to run a MediaPipe graph.
+#include <cstdlib>
 #include <fstream>
 #include <iostream>
 #include <map>
@@ -143,8 +144,9 @@ int main(int argc, char** argv) {
   ::mediapipe::Status run_status = RunMPPGraph();
   if (!run_status.ok()) {
     LOG(ERROR) << "Failed to run the graph: " << run_status.message();
+    return EXIT_FAILURE;
   } else {
     LOG(INFO) << "Success!";
   }
-  return 0;
+  return EXIT_SUCCESS;
 }

mediapipe/examples/desktop/youtube8m/README.md

@@ -1,6 +1,8 @@
 ### Steps to run the YouTube-8M feature extraction graph
 
-1.  Checkout the mediapipe repository.
+1.  Checkout the repository and follow
+    [the installation instructions](https://github.com/google/mediapipe/blob/master/mediapipe/docs/install.md)
+    to set up MediaPipe.
 
     ```bash
     git clone https://github.com/google/mediapipe.git

mediapipe/examples/desktop/youtube8m/extract_yt8m_features.cc

@@ -15,6 +15,7 @@
 // A simple main function to run a MediaPipe graph. Input side packets are read
 // from files provided via the command line and output side packets are written
 // to disk.
+#include <cstdlib>
 
 #include "absl/strings/str_split.h"
 #include "mediapipe/framework/calculator_framework.h"
@@ -128,8 +129,9 @@ int main(int argc, char** argv) {
   ::mediapipe::Status run_status = RunMPPGraph();
   if (!run_status.ok()) {
     LOG(ERROR) << "Failed to run the graph: " << run_status.message();
+    return EXIT_FAILURE;
   } else {
     LOG(INFO) << "Success!";
   }
-  return 0;
+  return EXIT_SUCCESS;
 }

mediapipe/examples/desktop/youtube8m/generate_input_sequence_example.py

@@ -21,7 +21,7 @@ import sys
 
 from absl import app
 from absl import flags
-import tensorflow as tf
+import tensorflow.compat.v1 as tf
 from mediapipe.util.sequence import media_sequence as ms
 
 FLAGS = flags.FLAGS

mediapipe/examples/desktop/youtube8m/generate_vggish_frozen_graph.py

@@ -24,7 +24,7 @@ import os
 import sys
 
 from absl import app
-import tensorflow as tf
+import tensorflow.compat.v1 as tf
 from tensorflow.python.tools import freeze_graph
 
 BASE_DIR = '/tmp/mediapipe/'