talksik/mediapipe
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Project import generated by Copybara.

Browse Source 
GitOrigin-RevId: f72a0f86c2c2acdb1920973c718a9e26ed3ec4b6
This commit is contained in:
MediaPipe Team
2020-06-08 12:08:33 -04:00
committed by chuoling
parent 5d028d923b
commit cd2b69d58c
315 changed files with 8162 additions and 7502 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mediapipe/examples/desktop/autoflip/BUILD
+3 -1
View File
@@ -30,7 +30,9 @@ mediapipe_cc_proto_library(
name = "autoflip_messages_cc_proto",
srcs = ["autoflip_messages.proto"],
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
visibility = ["//mediapipe/examples:__subpackages__"],
visibility = [
"//mediapipe/examples:__subpackages__",
],
deps = [":autoflip_messages_proto"],
)
mediapipe/examples/desktop/autoflip/autoflip_messages.proto
+19 -1
View File
@@ -73,7 +73,7 @@ message RectF {
// An image region of interest (eg a detected face or object), accompanied by an
// importance score.
// Next tag: 9
// Next tag: 10
message SalientRegion {
reserved 3;
// The bounding box for this region in the image.
@@ -98,7 +98,19 @@ message SalientRegion {
// If true, object cannot move in the output window (e.g. text would look
// strange moving around).
// TODO: this feature is not implemented, remove proto message.
optional bool requires_static_location = 7 [default = false];
// When used with ContentZoomingCalculator, this flag can be set in the
// SignalFusingCalculator indicating that areas outside of these detections
// can be cropped from the frame. When no salient regions have this flag set
// true, no zooming is performed. When one or more salient regions have this
// flag set true, the max zoom value will be used that keeps all
// “only_required” detections within view. The ContentZoomingCalculator
// currently supports zooming by finding the size of non-salient top/bottom
// borders regions and provides this information to the
// SceneCroppingCalculator for reframing.
optional bool only_required = 9 [default = false];
}
// Stores the message type, including standard types (face, object) and custom
@@ -180,4 +192,10 @@ message ExternalRenderFrame {
optional Color padding_color = 3;
// Timestamp in microseconds of this frame.
optional uint64 timestamp_us = 4;
// Target width of the cropped video in pixels. |render_to_location| is
// relative to this dimension.
optional int32 target_width = 5;
// Target height of the cropped video in pixels. |render_to_location| is
// relative to this dimension.
optional int32 target_height = 6;
}
mediapipe/examples/desktop/autoflip/calculators/BUILD
+66 -1
View File
@@ -16,7 +16,10 @@ load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library"
licenses(["notice"]) # Apache 2.0
package(default_visibility = ["//mediapipe/examples:__subpackages__"])
package(default_visibility = [
"//mediapipe/examples:__subpackages__",
"//mediapipe/viz:__subpackages__",
])
cc_library(
name = "border_detection_calculator",
@@ -51,6 +54,44 @@ mediapipe_cc_proto_library(
deps = [":border_detection_calculator_proto"],
)
cc_library(
name = "content_zooming_calculator",
srcs = ["content_zooming_calculator.cc"],
deps = [
":content_zooming_calculator_cc_proto",
"//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto",
"//mediapipe/examples/desktop/autoflip/quality:kinematic_path_solver",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/formats:image_frame_opencv",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
],
alwayslink = 1,
)
proto_library(
name = "content_zooming_calculator_proto",
srcs = ["content_zooming_calculator.proto"],
deps = [
"//mediapipe/examples/desktop/autoflip/quality:kinematic_path_solver_proto",
"//mediapipe/framework:calculator_proto",
],
)
mediapipe_cc_proto_library(
name = "content_zooming_calculator_cc_proto",
srcs = ["content_zooming_calculator.proto"],
cc_deps = [
"//mediapipe/examples/desktop/autoflip/quality:kinematic_path_solver_cc_proto",
"//mediapipe/framework:calculator_cc_proto",
],
visibility = ["//mediapipe/examples:__subpackages__"],
deps = [
":content_zooming_calculator_proto",
],
)
cc_test(
name = "border_detection_calculator_test",
srcs = [
@@ -75,6 +116,30 @@ cc_test(
],
)
cc_test(
name = "content_zooming_calculator_test",
srcs = [
"content_zooming_calculator_test.cc",
],
linkstatic = 1,
deps = [
":content_zooming_calculator",
":content_zooming_calculator_cc_proto",
"//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:calculator_runner",
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/formats:image_frame_opencv",
"//mediapipe/framework/port:benchmark",
"//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:opencv_core",
"//mediapipe/framework/port:parse_text_proto",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"@com_google_absl//absl/strings",
],
)
cc_library(
name = "video_filtering_calculator",
srcs = ["video_filtering_calculator.cc"],
mediapipe/examples/desktop/autoflip/calculators/content_zooming_calculator.cc
+280
View File
@@ -0,0 +1,280 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <algorithm>
#include <memory>
#include "mediapipe/examples/desktop/autoflip/autoflip_messages.pb.h"
#include "mediapipe/examples/desktop/autoflip/calculators/content_zooming_calculator.pb.h"
#include "mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/formats/image_frame_opencv.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/framework/port/status_builder.h"
constexpr char kVideoFrame[] = "VIDEO";
constexpr char kVideoSize[] = "VIDEO_SIZE";
constexpr char kDetectionSet[] = "DETECTIONS";
constexpr char kDetectedBorders[] = "BORDERS";
// Field-of-view (degrees) of the camera's x-axis (width).
// TODO: Parameterize FOV based on camera specs.
constexpr float kWidthFieldOfView = 60;
namespace mediapipe {
namespace autoflip {
// Content zooming calculator zooms in on content when a detection has
// "only_required" set true. It does this by computing the value of top/bottom
// borders to remove from the output and sends these to the
// SceneCroppingCalculator. When more than one detections are received the zoom
// box is calculated as the union of the detections. Typical applications
// include mobile makeover and autofliplive face reframing. Currently only
// supports y-dimension zooming.
class ContentZoomingCalculator : public CalculatorBase {
public:
ContentZoomingCalculator()
: initialized_(false), last_only_required_detection_(0) {}
~ContentZoomingCalculator() override {}
ContentZoomingCalculator(const ContentZoomingCalculator&) = delete;
ContentZoomingCalculator& operator=(const ContentZoomingCalculator&) = delete;
static ::mediapipe::Status GetContract(mediapipe::CalculatorContract* cc);
::mediapipe::Status Open(mediapipe::CalculatorContext* cc) override;
::mediapipe::Status Process(mediapipe::CalculatorContext* cc) override;
private:
// Converts bounds to tilt offset and height.
::mediapipe::Status ConvertToTiltZoom(float xmin, float xmax, float ymin,
float ymax, int* tilt_offset,
int* height);
ContentZoomingCalculatorOptions options_;
// Detection frame width/height.
int frame_height_;
int frame_width_;
// Path solver used to smooth top/bottom border crop values.
std::unique_ptr<KinematicPathSolver> path_solver_height_;
std::unique_ptr<KinematicPathSolver> path_solver_offset_;
// Are parameters initialized.
bool initialized_;
// Stores the time of the last "only_required" input.
int64 last_only_required_detection_;
// Border values of last message with detection.
int last_measured_height_;
int last_measured_y_offset_;
// Min border values.
float min_height_value_;
};
REGISTER_CALCULATOR(ContentZoomingCalculator);
::mediapipe::Status ContentZoomingCalculator::GetContract(
mediapipe::CalculatorContract* cc) {
RET_CHECK(
!(cc->Inputs().HasTag(kVideoFrame) && cc->Inputs().HasTag(kVideoSize)))
<< "Provide only VIDEO or VIDEO_SIZE, not both.";
if (cc->Inputs().HasTag(kVideoFrame)) {
cc->Inputs().Tag(kVideoFrame).Set<ImageFrame>();
} else if (cc->Inputs().HasTag(kVideoSize)) {
cc->Inputs().Tag(kVideoSize).Set<std::pair<int, int>>();
} else {
return ::mediapipe::UnknownErrorBuilder(MEDIAPIPE_LOC)
<< "Input VIDEO or VIDEO_SIZE must be provided.";
}
cc->Inputs().Tag(kDetectionSet).Set<DetectionSet>();
cc->Outputs().Tag(kDetectedBorders).Set<StaticFeatures>();
return ::mediapipe::OkStatus();
}
::mediapipe::Status ContentZoomingCalculator::Open(
mediapipe::CalculatorContext* cc) {
options_ = cc->Options<ContentZoomingCalculatorOptions>();
if (options_.has_kinematic_options()) {
return ::mediapipe::UnknownErrorBuilder(MEDIAPIPE_LOC)
<< "Deprecated kinematic_options was set, please set "
"kinematic_options_zoom and kinematic_options_tilt.";
}
if (options_.has_min_motion_to_reframe()) {
return ::mediapipe::UnknownErrorBuilder(MEDIAPIPE_LOC)
<< "Deprecated min_motion_to_reframe was set, please set "
"in kinematic_options_zoom and kinematic_options_tilt directly.";
}
return ::mediapipe::OkStatus();
}
::mediapipe::Status ContentZoomingCalculator::ConvertToTiltZoom(
float xmin, float xmax, float ymin, float ymax, int* tilt_offset,
int* height) {
// Find center of the y-axis offset (for tilt control).
float y_center = ymin + (ymax - ymin) / 2;
// Find size and apply scale factor to y-axis.
float fit_size = fmax((ymax - ymin) / options_.scale_factor(), xmax - xmin);
// Apply min zoom for cases where the target size is wider than input frame
// size.
fit_size = fmin(min_height_value_, fit_size);
// Prevent box from extending beyond the image.
if (y_center - fit_size / 2 < 0) {
y_center = fit_size / 2;
} else if (y_center + fit_size / 2 > 1) {
y_center = 1 - fit_size / 2;
}
// Scale to pixel coordinates.
*tilt_offset = frame_height_ * y_center;
*height = frame_height_ * fit_size;
return ::mediapipe::OkStatus();
}
namespace {
::mediapipe::Status UpdateRanges(const SalientRegion& region, float* xmin,
float* xmax, float* ymin, float* ymax) {
if (!region.has_location_normalized()) {
return ::mediapipe::UnknownErrorBuilder(MEDIAPIPE_LOC)
<< "SalientRegion did not have location normalized set.";
}
*xmin = fmin(*xmin, region.location_normalized().x());
*xmax = fmax(*xmax, region.location_normalized().x() +
region.location_normalized().width());
*ymin = fmin(*ymin, region.location_normalized().y());
*ymax = fmax(*ymax, region.location_normalized().y() +
region.location_normalized().height());
return ::mediapipe::OkStatus();
}
void MakeStaticFeatures(const int top_border, const int bottom_border,
const int frame_width, const int frame_height,
StaticFeatures* static_feature) {
auto border_top = static_feature->add_border();
border_top->set_relative_position(Border::TOP);
border_top->mutable_border_position()->set_x(0);
border_top->mutable_border_position()->set_y(0);
border_top->mutable_border_position()->set_width(frame_width);
border_top->mutable_border_position()->set_height(top_border);
auto border_bottom = static_feature->add_border();
border_bottom->set_relative_position(Border::BOTTOM);
border_bottom->mutable_border_position()->set_x(0);
border_bottom->mutable_border_position()->set_y(frame_height - bottom_border);
border_bottom->mutable_border_position()->set_width(frame_width);
border_bottom->mutable_border_position()->set_height(bottom_border);
}
} // namespace
::mediapipe::Status ContentZoomingCalculator::Process(
mediapipe::CalculatorContext* cc) {
if (cc->Inputs().HasTag(kVideoFrame)) {
cv::Mat frame = mediapipe::formats::MatView(
&cc->Inputs().Tag(kVideoFrame).Get<ImageFrame>());
frame_width_ = frame.cols;
frame_height_ = frame.rows;
} else if (cc->Inputs().HasTag(kVideoSize)) {
frame_width_ =
cc->Inputs().Tag(kVideoSize).Get<std::pair<int, int>>().first;
frame_height_ =
cc->Inputs().Tag(kVideoSize).Get<std::pair<int, int>>().second;
} else {
return ::mediapipe::UnknownErrorBuilder(MEDIAPIPE_LOC)
<< "Input VIDEO or VIDEO_SIZE must be provided.";
}
if (!initialized_) {
path_solver_height_ = std::make_unique<KinematicPathSolver>(
options_.kinematic_options_zoom(), 0, frame_height_,
static_cast<float>(frame_width_) / kWidthFieldOfView);
path_solver_offset_ = std::make_unique<KinematicPathSolver>(
options_.kinematic_options_tilt(), 0, frame_height_,
static_cast<float>(frame_width_) / kWidthFieldOfView);
min_height_value_ = 1.0;
// If target size is set and wider than input aspect, make sure to always
// crop the min required amount.
if (options_.has_target_size()) {
RET_CHECK_GT(options_.target_size().width(), 0)
<< "Provided target width not valid.";
RET_CHECK_GT(options_.target_size().height(), 0)
<< "Provided target height not valid.";
float input_aspect = frame_width_ / static_cast<float>(frame_height_);
float target_aspect = options_.target_size().width() /
static_cast<float>(options_.target_size().height());
min_height_value_ =
(input_aspect < target_aspect) ? input_aspect / target_aspect : 1.0;
}
last_measured_height_ = min_height_value_ * frame_height_;
last_measured_y_offset_ = frame_width_ / 2;
initialized_ = true;
}
auto detection_set = cc->Inputs().Tag(kDetectionSet).Get<DetectionSet>();
bool only_required_found = false;
// Compute the box that contains all "is_required" detections.
float xmin = 1, ymin = 1, xmax = 0, ymax = 0;
for (const auto& region : detection_set.detections()) {
if (!region.only_required()) {
continue;
}
only_required_found = true;
MP_RETURN_IF_ERROR(UpdateRanges(region, &xmin, &xmax, &ymin, &ymax));
}
// Convert bounds to tilt/zoom and in pixel coordinates.
int offset, height;
MP_RETURN_IF_ERROR(
ConvertToTiltZoom(xmin, xmax, ymin, ymax, &offset, &height));
if (only_required_found) {
// A only required detection was found.
last_only_required_detection_ = cc->InputTimestamp().Microseconds();
last_measured_height_ = height;
last_measured_y_offset_ = offset;
} else if (cc->InputTimestamp().Microseconds() -
last_only_required_detection_ >=
options_.us_before_zoomout()) {
// No only_require detections found within salient regions packets arriving
// since us_before_zoomout duration.
height = min_height_value_ * frame_height_;
offset = frame_height_ / 2;
} else {
// No only detection found but using last detection due to
// duration_before_zoomout_us setting.
height = last_measured_height_;
offset = last_measured_y_offset_;
}
// Compute smoothed camera paths.
MP_RETURN_IF_ERROR(path_solver_height_->AddObservation(
height, cc->InputTimestamp().Microseconds()));
MP_RETURN_IF_ERROR(path_solver_offset_->AddObservation(
offset, cc->InputTimestamp().Microseconds()));
int path_size;
MP_RETURN_IF_ERROR(path_solver_height_->GetState(&path_size));
int path_offset;
MP_RETURN_IF_ERROR(path_solver_offset_->GetState(&path_offset));
// Convert to top/bottom borders to remove.
int path_top = path_offset - path_size / 2;
int path_bottom = frame_height_ - (path_offset + path_size / 2);
// Transmit result downstream.
std::unique_ptr<StaticFeatures> features =
absl::make_unique<StaticFeatures>();
MakeStaticFeatures(path_top, path_bottom, frame_width_, frame_height_,
features.get());
cc->Outputs()
.Tag(kDetectedBorders)
.AddPacket(Adopt(features.release()).At(cc->InputTimestamp()));
return ::mediapipe::OkStatus();
}
} // namespace autoflip
} // namespace mediapipe
mediapipe/examples/desktop/autoflip/calculators/content_zooming_calculator.proto
+51
View File
@@ -0,0 +1,51 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto2";
package mediapipe.autoflip;
import "mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver.proto";
import "mediapipe/framework/calculator.proto";
message ContentZoomingCalculatorOptions {
extend mediapipe.CalculatorOptions {
optional ContentZoomingCalculatorOptions ext = 313091992;
}
// Amount the only required area should fill the image. When set to 1,
// reframing is done to remove content to the very edge of the salient region
// bounding box. When a smaller value is used, zooming will be done to fill
// this ratio of the frame.
optional float scale_factor = 1 [default = .9];
// Kinematic options for zooming.
optional KinematicOptions kinematic_options_zoom = 6;
// Kinematic options for tilt (y-axis reframing.)
optional KinematicOptions kinematic_options_tilt = 7;
// Duration (in MicroSeconds) before returning to fully zoomed out position
// when no "only_required" frames are received.
optional int64 us_before_zoomout = 9 [default = 1000000];
// Value of target output size, required to be set if different than input.
// Should match target_width and target_height in croppingcalculator.
message Size {
optional int64 width = 1;
optional int64 height = 2;
}
optional Size target_size = 8;
// Deprecated parameters
optional KinematicOptions kinematic_options = 2 [deprecated = true];
optional int64 min_motion_to_reframe = 4 [deprecated = true];
optional float min_vertical_zoom = 5 [default = 1, deprecated = true];
optional int64 frames_before_zoomout = 3 [default = 30, deprecated = true];
}
mediapipe/examples/desktop/autoflip/calculators/content_zooming_calculator_test.cc
+242
View File
@@ -0,0 +1,242 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <memory>
#include "mediapipe/examples/desktop/autoflip/autoflip_messages.pb.h"
#include "mediapipe/examples/desktop/autoflip/calculators/content_zooming_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_runner.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/formats/image_frame_opencv.h"
#include "mediapipe/framework/port/benchmark.h"
#include "mediapipe/framework/port/gmock.h"
#include "mediapipe/framework/port/gtest.h"
#include "mediapipe/framework/port/opencv_core_inc.h"
#include "mediapipe/framework/port/parse_text_proto.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/framework/port/status_matchers.h"
namespace mediapipe {
namespace autoflip {
namespace {
const char kConfigA[] = R"(
calculator: "ContentZoomingCalculator"
input_stream: "VIDEO:camera_frames"
input_stream: "DETECTIONS:detection_set"
output_stream: "BORDERS:borders"
)";
const char kConfigB[] = R"(
calculator: "ContentZoomingCalculator"
input_stream: "VIDEO:camera_frames"
input_stream: "DETECTIONS:detection_set"
output_stream: "BORDERS:borders"
options: {
[mediapipe.autoflip.ContentZoomingCalculatorOptions.ext]: {
target_size {
width: 1000
height: 500
}
}
}
)";
const char kConfigC[] = R"(
calculator: "ContentZoomingCalculator"
input_stream: "VIDEO_SIZE:size"
input_stream: "DETECTIONS:detection_set"
output_stream: "BORDERS:borders"
)";
void CheckBorder(const StaticFeatures& static_features, int width, int height,
int top_border, int bottom_border) {
ASSERT_EQ(2, static_features.border().size());
auto part = static_features.border(0);
EXPECT_EQ(part.border_position().x(), 0);
EXPECT_EQ(part.border_position().y(), 0);
EXPECT_EQ(part.border_position().width(), width);
EXPECT_EQ(part.border_position().height(), top_border);
EXPECT_EQ(Border::TOP, part.relative_position());
part = static_features.border(1);
EXPECT_EQ(part.border_position().x(), 0);
EXPECT_EQ(part.border_position().y(), height - bottom_border);
EXPECT_EQ(part.border_position().width(), width);
EXPECT_EQ(part.border_position().height(), bottom_border);
EXPECT_EQ(Border::BOTTOM, part.relative_position());
}
TEST(ContentZoomingCalculatorTest, ZoomTest) {
auto runner = ::absl::make_unique<CalculatorRunner>(
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfigA));
auto detection_set = std::make_unique<DetectionSet>();
auto* detection = detection_set->add_detections();
detection->set_only_required(true);
auto* location = detection->mutable_location_normalized();
location->set_height(.1);
location->set_width(.1);
location->set_x(.4);
location->set_y(.5);
auto input_frame =
::absl::make_unique<ImageFrame>(ImageFormat::SRGB, 1000, 1000);
runner->MutableInputs()->Tag("VIDEO").packets.push_back(
Adopt(input_frame.release()).At(Timestamp(0)));
runner->MutableInputs()
->Tag("DETECTIONS")
.packets.push_back(Adopt(detection_set.release()).At(Timestamp(0)));
// Run the calculator.
MP_ASSERT_OK(runner->Run());
const std::vector<Packet>& output_packets =
runner->Outputs().Tag("BORDERS").packets;
ASSERT_EQ(1, output_packets.size());
const auto& static_features = output_packets[0].Get<StaticFeatures>();
CheckBorder(static_features, 1000, 1000, 495, 395);
}
TEST(ContentZoomingCalculatorTest, MinAspectBorderValues) {
auto runner = ::absl::make_unique<CalculatorRunner>(
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfigB));
auto detection_set = std::make_unique<DetectionSet>();
auto* detection = detection_set->add_detections();
detection->set_only_required(true);
auto* location = detection->mutable_location_normalized();
location->set_height(1);
location->set_width(1);
location->set_x(0);
location->set_y(0);
auto input_frame =
::absl::make_unique<ImageFrame>(ImageFormat::SRGB, 1000, 1000);
runner->MutableInputs()->Tag("VIDEO").packets.push_back(
Adopt(input_frame.release()).At(Timestamp(0)));
runner->MutableInputs()
->Tag("DETECTIONS")
.packets.push_back(Adopt(detection_set.release()).At(Timestamp(0)));
// Run the calculator.
MP_ASSERT_OK(runner->Run());
const std::vector<Packet>& output_packets =
runner->Outputs().Tag("BORDERS").packets;
ASSERT_EQ(1, output_packets.size());
const auto& static_features = output_packets[0].Get<StaticFeatures>();
CheckBorder(static_features, 1000, 1000, 250, 250);
}
TEST(ContentZoomingCalculatorTest, TwoFacesWide) {
auto runner = ::absl::make_unique<CalculatorRunner>(
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfigA));
auto detection_set = std::make_unique<DetectionSet>();
auto* detection = detection_set->add_detections();
detection->set_only_required(true);
auto* location = detection->mutable_location_normalized();
location->set_height(.2);
location->set_width(.2);
location->set_x(.2);
location->set_y(.4);
location = detection->mutable_location_normalized();
location->set_height(.2);
location->set_width(.2);
location->set_x(.6);
location->set_y(.4);
auto input_frame =
::absl::make_unique<ImageFrame>(ImageFormat::SRGB, 1000, 1000);
runner->MutableInputs()->Tag("VIDEO").packets.push_back(
Adopt(input_frame.release()).At(Timestamp(0)));
runner->MutableInputs()
->Tag("DETECTIONS")
.packets.push_back(Adopt(detection_set.release()).At(Timestamp(0)));
// Run the calculator.
MP_ASSERT_OK(runner->Run());
const std::vector<Packet>& output_packets =
runner->Outputs().Tag("BORDERS").packets;
ASSERT_EQ(1, output_packets.size());
const auto& static_features = output_packets[0].Get<StaticFeatures>();
CheckBorder(static_features, 1000, 1000, 389, 389);
}
TEST(ContentZoomingCalculatorTest, NoDetectionOnInit) {
auto runner = ::absl::make_unique<CalculatorRunner>(
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfigA));
auto detection_set = std::make_unique<DetectionSet>();
auto input_frame =
::absl::make_unique<ImageFrame>(ImageFormat::SRGB, 1000, 1000);
runner->MutableInputs()->Tag("VIDEO").packets.push_back(
Adopt(input_frame.release()).At(Timestamp(0)));
runner->MutableInputs()
->Tag("DETECTIONS")
.packets.push_back(Adopt(detection_set.release()).At(Timestamp(0)));
// Run the calculator.
MP_ASSERT_OK(runner->Run());
const std::vector<Packet>& output_packets =
runner->Outputs().Tag("BORDERS").packets;
ASSERT_EQ(1, output_packets.size());
const auto& static_features = output_packets[0].Get<StaticFeatures>();
CheckBorder(static_features, 1000, 1000, 0, 0);
}
TEST(ContentZoomingCalculatorTest, ZoomTestPairSize) {
auto runner = ::absl::make_unique<CalculatorRunner>(
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfigC));
auto detection_set = std::make_unique<DetectionSet>();
auto* detection = detection_set->add_detections();
detection->set_only_required(true);
auto* location = detection->mutable_location_normalized();
location->set_height(.1);
location->set_width(.1);
location->set_x(.4);
location->set_y(.5);
auto input_size = ::absl::make_unique<std::pair<int, int>>(1000, 1000);
runner->MutableInputs()
->Tag("VIDEO_SIZE")
.packets.push_back(Adopt(input_size.release()).At(Timestamp(0)));
runner->MutableInputs()
->Tag("DETECTIONS")
.packets.push_back(Adopt(detection_set.release()).At(Timestamp(0)));
// Run the calculator.
MP_ASSERT_OK(runner->Run());
const std::vector<Packet>& output_packets =
runner->Outputs().Tag("BORDERS").packets;
ASSERT_EQ(1, output_packets.size());
const auto& static_features = output_packets[0].Get<StaticFeatures>();
CheckBorder(static_features, 1000, 1000, 495, 395);
}
} // namespace
} // namespace autoflip
} // namespace mediapipe
mediapipe/examples/desktop/autoflip/calculators/face_to_region_calculator.cc
+25 -7
View File
@@ -80,7 +80,9 @@ FaceToRegionCalculator::FaceToRegionCalculator() {}
::mediapipe::Status FaceToRegionCalculator::GetContract(
mediapipe::CalculatorContract* cc) {
cc->Inputs().Tag("VIDEO").Set<ImageFrame>();
if (cc->Inputs().HasTag("VIDEO")) {
cc->Inputs().Tag("VIDEO").Set<ImageFrame>();
}
cc->Inputs().Tag("FACES").Set<std::vector<mediapipe::Detection>>();
cc->Outputs().Tag("REGIONS").Set<DetectionSet>();
return ::mediapipe::OkStatus();
@@ -89,6 +91,17 @@ FaceToRegionCalculator::FaceToRegionCalculator() {}
::mediapipe::Status FaceToRegionCalculator::Open(
mediapipe::CalculatorContext* cc) {
options_ = cc->Options<FaceToRegionCalculatorOptions>();
if (!cc->Inputs().HasTag("VIDEO")) {
RET_CHECK(!options_.use_visual_scorer())
<< "VIDEO input must be provided when using visual_scorer.";
RET_CHECK(!options_.export_individual_face_landmarks())
<< "VIDEO input must be provided when export_individual_face_landmarks "
"is set true.";
RET_CHECK(!options_.export_bbox_from_landmarks())
<< "VIDEO input must be provided when export_bbox_from_landmarks "
"is set true.";
}
scorer_ = absl::make_unique<VisualScorer>(options_.scorer_options());
frame_width_ = -1;
frame_height_ = -1;
@@ -135,14 +148,19 @@ void FaceToRegionCalculator::ExtendSalientRegionWithPoint(
::mediapipe::Status FaceToRegionCalculator::Process(
mediapipe::CalculatorContext* cc) {
if (cc->Inputs().Tag("VIDEO").Value().IsEmpty()) {
return ::mediapipe::UnknownErrorBuilder(MEDIAPIPE_LOC) << "No VIDEO input.";
if (cc->Inputs().HasTag("VIDEO") &&
cc->Inputs().Tag("VIDEO").Value().IsEmpty()) {
return ::mediapipe::UnknownErrorBuilder(MEDIAPIPE_LOC)
<< "No VIDEO input at time " << cc->InputTimestamp().Seconds();
}
cv::Mat frame =
mediapipe::formats::MatView(&cc->Inputs().Tag("VIDEO").Get<ImageFrame>());
frame_width_ = frame.cols;
frame_height_ = frame.rows;
cv::Mat frame;
if (cc->Inputs().HasTag("VIDEO")) {
frame = mediapipe::formats::MatView(
&cc->Inputs().Tag("VIDEO").Get<ImageFrame>());
frame_width_ = frame.cols;
frame_height_ = frame.rows;
}
auto region_set = ::absl::make_unique<DetectionSet>();
if (!cc->Inputs().Tag("FACES").Value().IsEmpty()) {
mediapipe/examples/desktop/autoflip/calculators/face_to_region_calculator_test.cc
+96 -24
View File
@@ -40,6 +40,12 @@ const char kConfig[] = R"(
output_stream: "REGIONS:regions"
)";
const char kConfigNoVideo[] = R"(
calculator: "FaceToRegionCalculator"
input_stream: "FACES:faces"
output_stream: "REGIONS:regions"
)";
const char kFace1[] = R"(location_data {
format: RELATIVE_BOUNDING_BOX
relative_bounding_box {
@@ -88,13 +94,15 @@ const char kFace3[] = R"(location_data {
relative_keypoints { x: 0 y: 0 }
})";
void SetInputs(CalculatorRunner* runner,
const std::vector<std::string>& faces) {
void SetInputs(const std::vector<std::string>& faces, const bool include_video,
CalculatorRunner* runner) {
// Setup an input video frame.
auto input_frame =
::absl::make_unique<ImageFrame>(ImageFormat::SRGB, 800, 600);
runner->MutableInputs()->Tag("VIDEO").packets.push_back(
Adopt(input_frame.release()).At(Timestamp::PostStream()));
if (include_video) {
auto input_frame =
::absl::make_unique<ImageFrame>(ImageFormat::SRGB, 800, 600);
runner->MutableInputs()->Tag("VIDEO").packets.push_back(
Adopt(input_frame.release()).At(Timestamp::PostStream()));
}
// Setup two faces as input.
auto input_faces = ::absl::make_unique<std::vector<Detection>>();
// A face with landmarks.
@@ -105,30 +113,31 @@ void SetInputs(CalculatorRunner* runner,
Adopt(input_faces.release()).At(Timestamp::PostStream()));
}
CalculatorGraphConfig::Node MakeConfig(bool whole_face, bool landmarks,
bool bb_from_landmarks) {
auto config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfig);
CalculatorGraphConfig::Node MakeConfig(std::string base_config, bool whole_face,
bool landmarks, bool bb_from_landmarks,
bool visual_scoring) {
auto config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(base_config);
config.mutable_options()
->MutableExtension(FaceToRegionCalculatorOptions::ext)
->set_export_whole_face(whole_face);
config.mutable_options()
->MutableExtension(FaceToRegionCalculatorOptions::ext)
->set_export_individual_face_landmarks(landmarks);
config.mutable_options()
->MutableExtension(FaceToRegionCalculatorOptions::ext)
->set_export_bbox_from_landmarks(bb_from_landmarks);
config.mutable_options()
->MutableExtension(FaceToRegionCalculatorOptions::ext)
->set_use_visual_scorer(visual_scoring);
return config;
}
TEST(FaceToRegionCalculatorTest, FaceFullTypeSize) {
// Setup test
auto runner =
::absl::make_unique<CalculatorRunner>(MakeConfig(true, false, false));
SetInputs(runner.get(), {kFace1, kFace2});
auto runner = ::absl::make_unique<CalculatorRunner>(
MakeConfig(kConfig, true, false, false, true));
SetInputs({kFace1, kFace2}, true, runner.get());
// Run the calculator.
MP_ASSERT_OK(runner->Run());
@@ -159,9 +168,9 @@ TEST(FaceToRegionCalculatorTest, FaceFullTypeSize) {
TEST(FaceToRegionCalculatorTest, FaceLandmarksTypeSize) {
// Setup test
auto runner =
::absl::make_unique<CalculatorRunner>(MakeConfig(false, true, false));
SetInputs(runner.get(), {kFace1});
auto runner = ::absl::make_unique<CalculatorRunner>(
MakeConfig(kConfig, false, true, false, true));
SetInputs({kFace1}, true, runner.get());
// Run the calculator.
MP_ASSERT_OK(runner->Run());
@@ -190,9 +199,9 @@ TEST(FaceToRegionCalculatorTest, FaceLandmarksTypeSize) {
TEST(FaceToRegionCalculatorTest, FaceLandmarksBox) {
// Setup test
auto runner =
::absl::make_unique<CalculatorRunner>(MakeConfig(false, false, true));
SetInputs(runner.get(), {kFace1});
auto runner = ::absl::make_unique<CalculatorRunner>(
MakeConfig(kConfig, false, false, true, true));
SetInputs({kFace1}, true, runner.get());
// Run the calculator.
MP_ASSERT_OK(runner->Run());
@@ -225,9 +234,9 @@ TEST(FaceToRegionCalculatorTest, FaceLandmarksBox) {
TEST(FaceToRegionCalculatorTest, FaceScore) {
// Setup test
auto runner =
::absl::make_unique<CalculatorRunner>(MakeConfig(true, false, false));
SetInputs(runner.get(), {kFace3});
auto runner = ::absl::make_unique<CalculatorRunner>(
MakeConfig(kConfig, true, false, false, true));
SetInputs({kFace3}, true, runner.get());
// Run the calculator.
MP_ASSERT_OK(runner->Run());
@@ -242,6 +251,69 @@ TEST(FaceToRegionCalculatorTest, FaceScore) {
EXPECT_FLOAT_EQ(landmark_1.score(), 0.25);
}
TEST(FaceToRegionCalculatorTest, FaceNoVideoVisualScoreFail) {
// Setup test
auto runner = ::absl::make_unique<CalculatorRunner>(
MakeConfig(kConfigNoVideo, true, false, false, true));
SetInputs({kFace3}, false, runner.get());
// Run the calculator.
ASSERT_FALSE(runner->Run().ok());
}
TEST(FaceToRegionCalculatorTest, FaceNoVideoLandmarksFail) {
// Setup test
auto runner = ::absl::make_unique<CalculatorRunner>(
MakeConfig(kConfigNoVideo, false, true, false, false));
SetInputs({kFace3}, false, runner.get());
// Run the calculator.
ASSERT_FALSE(runner->Run().ok());
}
TEST(FaceToRegionCalculatorTest, FaceNoVideoBBLandmarksFail) {
// Setup test
auto runner = ::absl::make_unique<CalculatorRunner>(
MakeConfig(kConfigNoVideo, false, false, true, false));
SetInputs({kFace3}, false, runner.get());
// Run the calculator.
ASSERT_FALSE(runner->Run().ok());
}
TEST(FaceToRegionCalculatorTest, FaceNoVideoPass) {
// Setup test
auto runner = ::absl::make_unique<CalculatorRunner>(
MakeConfig(kConfigNoVideo, true, false, false, false));
SetInputs({kFace1, kFace2}, false, runner.get());
// Run the calculator.
MP_ASSERT_OK(runner->Run());
// Check the output regions.
const std::vector<Packet>& output_packets =
runner->Outputs().Tag("REGIONS").packets;
ASSERT_EQ(1, output_packets.size());
const auto& regions = output_packets[0].Get<DetectionSet>();
ASSERT_EQ(2, regions.detections().size());
auto face_1 = regions.detections(0);
EXPECT_EQ(face_1.signal_type().standard(), SignalType::FACE_FULL);
EXPECT_FLOAT_EQ(face_1.location_normalized().x(), 0);
EXPECT_FLOAT_EQ(face_1.location_normalized().y(), 0.003333);
EXPECT_FLOAT_EQ(face_1.location_normalized().width(), 0.12125);
EXPECT_FLOAT_EQ(face_1.location_normalized().height(), 0.33333);
EXPECT_FLOAT_EQ(face_1.score(), 1);
auto face_2 = regions.detections(1);
EXPECT_EQ(face_2.signal_type().standard(), SignalType::FACE_FULL);
EXPECT_FLOAT_EQ(face_2.location_normalized().x(), 0.0025);
EXPECT_FLOAT_EQ(face_2.location_normalized().y(), 0.005);
EXPECT_FLOAT_EQ(face_2.location_normalized().width(), 0.25);
EXPECT_FLOAT_EQ(face_2.location_normalized().height(), 0.5);
EXPECT_FLOAT_EQ(face_2.score(), 1);
}
} // namespace
} // namespace autoflip
} // namespace mediapipe
mediapipe/examples/desktop/autoflip/calculators/scene_cropping_calculator.cc
+242 -129
View File
@@ -24,6 +24,7 @@
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/formats/image_frame_opencv.h"
#include "mediapipe/framework/port/canonical_errors.h"
#include "mediapipe/framework/port/opencv_core_inc.h"
#include "mediapipe/framework/port/opencv_imgproc_inc.h"
#include "mediapipe/framework/port/parse_text_proto.h"
@@ -35,6 +36,7 @@ namespace mediapipe {
namespace autoflip {
constexpr char kInputVideoFrames[] = "VIDEO_FRAMES";
constexpr char kInputVideoSize[] = "VIDEO_SIZE";
constexpr char kInputKeyFrames[] = "KEY_FRAMES";
constexpr char kInputDetections[] = "DETECTION_FEATURES";
constexpr char kInputStaticFeatures[] = "STATIC_FEATURES";
@@ -49,8 +51,17 @@ constexpr char kAspectRatio[] = "EXTERNAL_ASPECT_RATIO";
// SALIENT_POINT_FRAME_VIZ_FRAMES can only be enabled when CROPPED_FRAMES is
// enabled.
constexpr char kOutputCroppedFrames[] = "CROPPED_FRAMES";
// Shows detections on key frames. Any static borders will be removed from the
// output frame.
constexpr char kOutputKeyFrameCropViz[] = "KEY_FRAME_CROP_REGION_VIZ_FRAMES";
// Shows x/y (raw unsmoothed) cropping and focus points. Any static borders
// will be removed from the output frame.
constexpr char kOutputFocusPointFrameViz[] = "SALIENT_POINT_FRAME_VIZ_FRAMES";
// Shows final smoothed cropping and a focused area of the camera. Any static
// borders will remain and be shown in grey. Output frame will match input
// frame size.
constexpr char kOutputFramingAndDetections[] = "FRAMING_DETECTIONS_VIZ_FRAMES";
// Final summary of cropping.
constexpr char kOutputSummary[] = "CROPPING_SUMMARY";
// External rendering outputs
@@ -65,7 +76,12 @@ constexpr char kExternalRenderingFullVid[] = "EXTERNAL_RENDERING_FULL_VID";
if (cc->InputSidePackets().HasTag(kAspectRatio)) {
cc->InputSidePackets().Tag(kAspectRatio).Set<std::string>();
}
cc->Inputs().Tag(kInputVideoFrames).Set<ImageFrame>();
if (cc->Inputs().HasTag(kInputVideoFrames)) {
cc->Inputs().Tag(kInputVideoFrames).Set<ImageFrame>();
}
if (cc->Inputs().HasTag(kInputVideoSize)) {
cc->Inputs().Tag(kInputVideoSize).Set<std::pair<int, int>>();
}
if (cc->Inputs().HasTag(kInputKeyFrames)) {
cc->Inputs().Tag(kInputKeyFrames).Set<ImageFrame>();
}
@@ -73,7 +89,9 @@ constexpr char kExternalRenderingFullVid[] = "EXTERNAL_RENDERING_FULL_VID";
if (cc->Inputs().HasTag(kInputStaticFeatures)) {
cc->Inputs().Tag(kInputStaticFeatures).Set<StaticFeatures>();
}
cc->Inputs().Tag(kInputShotBoundaries).Set<bool>();
if (cc->Inputs().HasTag(kInputShotBoundaries)) {
cc->Inputs().Tag(kInputShotBoundaries).Set<bool>();
}
if (cc->Outputs().HasTag(kOutputCroppedFrames)) {
cc->Outputs().Tag(kOutputCroppedFrames).Set<ImageFrame>();
@@ -84,6 +102,12 @@ constexpr char kExternalRenderingFullVid[] = "EXTERNAL_RENDERING_FULL_VID";
"CROPPED_FRAMES is specified.";
cc->Outputs().Tag(kOutputKeyFrameCropViz).Set<ImageFrame>();
}
if (cc->Outputs().HasTag(kOutputFramingAndDetections)) {
RET_CHECK(cc->Outputs().HasTag(kOutputCroppedFrames))
<< "FRAMING_DETECTIONS_VIZ_FRAMES can only be used when "
"CROPPED_FRAMES is specified.";
cc->Outputs().Tag(kOutputFramingAndDetections).Set<ImageFrame>();
}
if (cc->Outputs().HasTag(kOutputFocusPointFrameViz)) {
RET_CHECK(cc->Outputs().HasTag(kOutputCroppedFrames))
<< "SALIENT_POINT_FRAME_VIZ_FRAMES can only be used when "
@@ -101,6 +125,13 @@ constexpr char kExternalRenderingFullVid[] = "EXTERNAL_RENDERING_FULL_VID";
.Tag(kExternalRenderingFullVid)
.Set<std::vector<ExternalRenderFrame>>();
}
RET_CHECK(cc->Inputs().HasTag(kInputVideoFrames) ^
cc->Inputs().HasTag(kInputVideoSize))
<< "VIDEO_FRAMES or VIDEO_SIZE must be set and not both.";
RET_CHECK(!(cc->Inputs().HasTag(kInputVideoSize) &&
cc->Inputs().HasTag(kOutputCroppedFrames)))
<< "CROPPED_FRAMES (internal cropping) has been set as an output without "
"VIDEO_FRAMES (video data) input.";
RET_CHECK(cc->Outputs().HasTag(kExternalRenderingPerFrame) ||
cc->Outputs().HasTag(kExternalRenderingFullVid) ||
cc->Outputs().HasTag(kOutputCroppedFrames))
@@ -128,7 +159,12 @@ constexpr char kExternalRenderingFullVid[] = "EXTERNAL_RENDERING_FULL_VID";
RET_CHECK(overlay_opacity_ >= 0.0 && overlay_opacity_ <= 1.0)
<< "Overlay opacity " << overlay_opacity_ << " is not in [0, 1].";
scene_cropper_ = absl::make_unique<SceneCropper>();
// Set default camera model to polynomial_path_solver.
if (!options_.camera_motion_options().has_kinematic_options()) {
options_.mutable_camera_motion_options()
->mutable_polynomial_path_solver()
->set_prior_frame_buffer_size(options_.prior_frame_buffer_size());
}
if (cc->Outputs().HasTag(kOutputSummary)) {
summary_ = absl::make_unique<VideoCroppingSummary>();
}
@@ -137,6 +173,8 @@ constexpr char kExternalRenderingFullVid[] = "EXTERNAL_RENDERING_FULL_VID";
absl::make_unique<std::vector<ExternalRenderFrame>>();
}
should_perform_frame_cropping_ = cc->Outputs().HasTag(kOutputCroppedFrames);
scene_camera_motion_analyzer_ = absl::make_unique<SceneCameraMotionAnalyzer>(
options_.scene_camera_motion_analyzer_options());
return ::mediapipe::OkStatus();
}
@@ -182,44 +220,59 @@ void ConstructExternalRenderMessage(
padding_color_message->set_b(padding_color[2]);
external_render_message->set_timestamp_us(timestamp_us);
}
double GetRatio(int width, int height) {
return static_cast<double>(width) / height;
}
int RoundToEven(float value) {
int rounded_value = std::round(value);
if (rounded_value % 2 == 1) {
rounded_value = std::max(2, rounded_value - 1);
}
return rounded_value;
}
} // namespace
::mediapipe::Status SceneCroppingCalculator::Process(
::mediapipe::Status SceneCroppingCalculator::InitializeSceneCroppingCalculator(
::mediapipe::CalculatorContext* cc) {
// Sets frame dimension and format.
if (frame_width_ < 0 &&
!cc->Inputs().Tag(kInputVideoFrames).Value().IsEmpty()) {
if (cc->Inputs().HasTag(kInputVideoFrames)) {
const auto& frame = cc->Inputs().Tag(kInputVideoFrames).Get<ImageFrame>();
frame_width_ = frame.Width();
RET_CHECK_GT(frame_width_, 0) << "Input frame width is non-positive.";
frame_height_ = frame.Height();
RET_CHECK_GT(frame_height_, 0) << "Input frame height is non-positive.";
frame_format_ = frame.Format();
target_width_ = options_.target_width();
target_height_ = options_.target_height();
if (cc->InputSidePackets().HasTag(kInputExternalSettings)) {
auto conversion_options = ParseTextProtoOrDie<ConversionOptions>(
cc->InputSidePackets()
.Tag(kInputExternalSettings)
.Get<std::string>());
target_width_ = conversion_options.target_width();
target_height_ = conversion_options.target_height();
}
target_aspect_ratio_ = static_cast<double>(target_width_) / target_height_;
RET_CHECK_NE(options_.target_size_type(),
SceneCroppingCalculatorOptions::UNKNOWN)
<< "TargetSizeType not set properly.";
// Resets target size if keep original height or width.
if (options_.target_size_type() ==
SceneCroppingCalculatorOptions::KEEP_ORIGINAL_HEIGHT) {
target_height_ = frame_height_;
target_width_ = std::round(target_height_ * target_aspect_ratio_);
} else if (options_.target_size_type() ==
SceneCroppingCalculatorOptions::KEEP_ORIGINAL_WIDTH) {
target_width_ = frame_width_;
target_height_ = std::round(target_width_ / target_aspect_ratio_);
} else if (options_.target_size_type() ==
SceneCroppingCalculatorOptions::MAXIMIZE_TARGET_DIMENSION) {
} else if (cc->Inputs().HasTag(kInputVideoSize)) {
frame_width_ =
cc->Inputs().Tag(kInputVideoSize).Get<std::pair<int, int>>().first;
frame_height_ =
cc->Inputs().Tag(kInputVideoSize).Get<std::pair<int, int>>().second;
} else {
return ::mediapipe::UnknownErrorBuilder(MEDIAPIPE_LOC)
<< "Input VIDEO or VIDEO_SIZE must be provided.";
}
RET_CHECK_GT(frame_height_, 0) << "Input frame height is non-positive.";
RET_CHECK_GT(frame_width_, 0) << "Input frame width is non-positive.";
// Calculate target width and height.
switch (options_.target_size_type()) {
case SceneCroppingCalculatorOptions::KEEP_ORIGINAL_HEIGHT:
RET_CHECK(options_.has_target_width() && options_.has_target_height())
<< "Target width and height have to be specified.";
target_height_ = RoundToEven(frame_height_);
target_width_ =
RoundToEven(target_height_ * GetRatio(options_.target_width(),
options_.target_height()));
break;
case SceneCroppingCalculatorOptions::KEEP_ORIGINAL_WIDTH:
RET_CHECK(options_.has_target_width() && options_.has_target_height())
<< "Target width and height have to be specified.";
target_width_ = RoundToEven(frame_width_);
target_height_ =
RoundToEven(target_width_ / GetRatio(options_.target_width(),
options_.target_height()));
break;
case SceneCroppingCalculatorOptions::MAXIMIZE_TARGET_DIMENSION: {
RET_CHECK(cc->InputSidePackets().HasTag(kAspectRatio))
<< "MAXIMIZE_TARGET_DIMENSION is set without an "
"external_aspect_ratio";
@@ -228,49 +281,81 @@ void ConstructExternalRenderMessage(
cc->InputSidePackets().Tag(kAspectRatio).Get<std::string>(),
&requested_aspect_ratio));
const double original_aspect_ratio =
static_cast<double>(frame_width_) / frame_height_;
GetRatio(frame_width_, frame_height_);
if (original_aspect_ratio > requested_aspect_ratio) {
target_height_ = frame_height_;
target_width_ = std::round(target_height_ * requested_aspect_ratio);
target_height_ = RoundToEven(frame_height_);
target_width_ = RoundToEven(target_height_ * requested_aspect_ratio);
} else {
target_width_ = frame_width_;
target_height_ = std::round(target_width_ / requested_aspect_ratio);
target_width_ = RoundToEven(frame_width_);
target_height_ = RoundToEven(target_width_ / requested_aspect_ratio);
}
break;
}
// Makes sure that target size is even if keep original width or height.
if (options_.target_size_type() !=
SceneCroppingCalculatorOptions::USE_TARGET_DIMENSION) {
if (target_width_ % 2 == 1) {
target_width_ = std::max(2, target_width_ - 1);
}
if (target_height_ % 2 == 1) {
target_height_ = std::max(2, target_height_ - 1);
}
target_aspect_ratio_ =
static_cast<double>(target_width_) / target_height_;
}
// Set keyframe width/height for feature upscaling (overwritten by keyframe
// input if provided).
if (options_.has_video_features_width() &&
options_.has_video_features_height()) {
key_frame_width_ = options_.video_features_width();
key_frame_height_ = options_.video_features_height();
} else if (!cc->Inputs().HasTag(kInputKeyFrames)) {
key_frame_width_ = frame_width_;
key_frame_height_ = frame_height_;
}
// Check provided dimensions.
RET_CHECK_GT(target_width_, 0) << "Target width is non-positive.";
RET_CHECK_NE(target_width_ % 2, 1)
<< "Target width cannot be odd, because encoder expects dimension "
"values to be even.";
RET_CHECK_GT(target_height_, 0) << "Target height is non-positive.";
RET_CHECK_NE(target_height_ % 2, 1)
<< "Target height cannot be odd, because encoder expects dimension "
"values to be even.";
case SceneCroppingCalculatorOptions::USE_TARGET_DIMENSION:
RET_CHECK(options_.has_target_width() && options_.has_target_height())
<< "Target width and height have to be specified.";
target_width_ = options_.target_width();
target_height_ = options_.target_height();
break;
case SceneCroppingCalculatorOptions::KEEP_ORIGINAL_DIMENSION:
target_width_ = frame_width_;
target_height_ = frame_height_;
break;
case SceneCroppingCalculatorOptions::UNKNOWN:
return mediapipe::InvalidArgumentError(
"target_size_type not set properly.");
}
target_aspect_ratio_ = GetRatio(target_width_, target_height_);
// Set keyframe width/height for feature upscaling.
RET_CHECK(!(cc->Inputs().HasTag(kInputKeyFrames) &&
(options_.has_video_features_width() ||
options_.has_video_features_height())))
<< "Key frame size must be defined by either providing the input stream "
"KEY_FRAMES or setting video_features_width/video_features_height as "
"calculator options. Both methods cannot be used together.";
if (options_.has_video_features_width() &&
options_.has_video_features_height()) {
key_frame_width_ = options_.video_features_width();
key_frame_height_ = options_.video_features_height();
} else if (!cc->Inputs().HasTag(kInputKeyFrames)) {
key_frame_width_ = frame_width_;
key_frame_height_ = frame_height_;
}
// Check provided dimensions.
RET_CHECK_GT(target_width_, 0) << "Target width is non-positive.";
// TODO: it seems this check is too strict and maybe limiting,
// considering the receiver of frames can be something other than encoder.
RET_CHECK_NE(target_width_ % 2, 1)
<< "Target width cannot be odd, because encoder expects dimension "
"values to be even.";
RET_CHECK_GT(target_height_, 0) << "Target height is non-positive.";
RET_CHECK_NE(target_height_ % 2, 1)
<< "Target height cannot be odd, because encoder expects dimension "
"values to be even.";
scene_cropper_ = absl::make_unique<SceneCropper>(
options_.camera_motion_options(), frame_width_, frame_height_);
return ::mediapipe::OkStatus();
}
bool HasFrameSignal(::mediapipe::CalculatorContext* cc) {
if (cc->Inputs().HasTag(kInputVideoFrames)) {
return !cc->Inputs().Tag(kInputVideoFrames).Value().IsEmpty();
}
return !cc->Inputs().Tag(kInputVideoSize).Value().IsEmpty();
}
::mediapipe::Status SceneCroppingCalculator::Process(
::mediapipe::CalculatorContext* cc) {
// Sets frame dimension and initializes scenecroppingcalculator on first video
// frame.
if (frame_width_ < 0) {
MP_RETURN_IF_ERROR(InitializeSceneCroppingCalculator(cc));
}
// Sets key frame dimension.
// Sets key frame dimension on first keyframe.
if (cc->Inputs().HasTag(kInputKeyFrames) &&
!cc->Inputs().Tag(kInputKeyFrames).Value().IsEmpty() &&
key_frame_width_ < 0) {
@@ -281,21 +366,18 @@ void ConstructExternalRenderMessage(
// Processes a scene when shot boundary or buffer is full.
bool is_end_of_scene = false;
if (!cc->Inputs().Tag(kInputShotBoundaries).Value().IsEmpty()) {
if (cc->Inputs().HasTag(kInputShotBoundaries) &&
!cc->Inputs().Tag(kInputShotBoundaries).Value().IsEmpty()) {
is_end_of_scene = cc->Inputs().Tag(kInputShotBoundaries).Get<bool>();
}
const bool force_buffer_flush =
scene_frame_timestamps_.size() >= options_.max_scene_size();
if (!scene_frame_timestamps_.empty() &&
(is_end_of_scene || force_buffer_flush)) {
if (!scene_frame_timestamps_.empty() && (is_end_of_scene)) {
continue_last_scene_ = false;
MP_RETURN_IF_ERROR(ProcessScene(is_end_of_scene, cc));
}
// Saves frame and timestamp and whether it is a key frame.
if (!cc->Inputs().Tag(kInputVideoFrames).Value().IsEmpty()) {
LOG_EVERY_N(ERROR, 10)
<< "------------------------ (Breathing) Time(s): "
<< cc->Inputs().Tag(kInputVideoFrames).Value().Timestamp().Seconds();
if (HasFrameSignal(cc)) {
// Only buffer frames if |should_perform_frame_cropping_| is true.
if (should_perform_frame_cropping_) {
const auto& frame = cc->Inputs().Tag(kInputVideoFrames).Get<ImageFrame>();
@@ -328,6 +410,13 @@ void ConstructExternalRenderMessage(
static_features_timestamps_.push_back(cc->InputTimestamp().Value());
}
const bool force_buffer_flush =
scene_frame_timestamps_.size() >= options_.max_scene_size();
if (!scene_frame_timestamps_.empty() && force_buffer_flush) {
MP_RETURN_IF_ERROR(ProcessScene(is_end_of_scene, cc));
continue_last_scene_ = true;
}
return ::mediapipe::OkStatus();
}
@@ -352,7 +441,7 @@ void ConstructExternalRenderMessage(
// TODO: split this function into two, one for calculating the border
// sizes, the other for the actual removal of borders from the frames.
::mediapipe::Status SceneCroppingCalculator::RemoveStaticBorders(
int* top_border_size, int* bottom_border_size) {
CalculatorContext* cc, int* top_border_size, int* bottom_border_size) {
*top_border_size = 0;
*bottom_border_size = 0;
MP_RETURN_IF_ERROR(ComputeSceneStaticBordersSize(
@@ -363,6 +452,13 @@ void ConstructExternalRenderMessage(
effective_frame_height_ =
frame_height_ - top_border_distance_ - bottom_border_distance;
// Store shallow copy of the original frames for debug display if required
// before static areas are removed.
if (cc->Outputs().HasTag(kOutputFramingAndDetections)) {
raw_scene_frames_or_empty_ = {scene_frames_or_empty_.begin(),
scene_frames_or_empty_.end()};
}
if (top_border_distance_ > 0 || bottom_border_distance > 0) {
VLOG(1) << "Remove top border " << top_border_distance_ << " bottom border "
<< bottom_border_distance;
@@ -442,8 +538,8 @@ void SceneCroppingCalculator::FilterKeyFrameInfo() {
// Removes any static borders.
int top_static_border_size, bottom_static_border_size;
MP_RETURN_IF_ERROR(
RemoveStaticBorders(&top_static_border_size, &bottom_static_border_size));
MP_RETURN_IF_ERROR(RemoveStaticBorders(cc, &top_static_border_size,
&bottom_static_border_size));
// Decides if solid background color padding is possible and sets up color
// interpolation functions in CIELAB. Uses linear interpolation by default.
@@ -453,7 +549,8 @@ void SceneCroppingCalculator::FilterKeyFrameInfo() {
&has_solid_background_, &background_color_l_function_,
&background_color_a_function_, &background_color_b_function_));
// Computes key frame crop regions.
// Computes key frame crop regions and moves information from raw
// key_frame_infos_ to key_frame_crop_results.
MP_RETURN_IF_ERROR(InitializeFrameCropRegionComputer());
const int num_key_frames = key_frame_infos_.size();
std::vector<KeyFrameCropResult> key_frame_crop_results(num_key_frames);
@@ -462,20 +559,15 @@ void SceneCroppingCalculator::FilterKeyFrameInfo() {
key_frame_infos_[i], &key_frame_crop_results[i]));
}
// Analyzes scene camera motion and generates FocusPointFrames.
auto analyzer_options = options_.scene_camera_motion_analyzer_options();
analyzer_options.set_allow_sweeping(analyzer_options.allow_sweeping() &&
!has_solid_background_);
scene_camera_motion_analyzer_ =
absl::make_unique<SceneCameraMotionAnalyzer>(analyzer_options);
SceneKeyFrameCropSummary scene_summary;
std::vector<FocusPointFrame> focus_point_frames;
SceneCameraMotion scene_camera_motion;
MP_RETURN_IF_ERROR(
scene_camera_motion_analyzer_->AnalyzeSceneAndPopulateFocusPointFrames(
key_frame_infos_, key_frame_crop_options_, key_frame_crop_results,
frame_width_, effective_frame_height_, scene_frame_timestamps_,
&scene_summary, &focus_point_frames, &scene_camera_motion));
key_frame_crop_options_, key_frame_crop_results, frame_width_,
effective_frame_height_, scene_frame_timestamps_,
has_solid_background_, &scene_summary, &focus_point_frames,
&scene_camera_motion));
// Crops scene frames.
std::vector<cv::Mat> cropped_frames;
@@ -485,26 +577,28 @@ void SceneCroppingCalculator::FilterKeyFrameInfo() {
should_perform_frame_cropping_ ? &cropped_frames : nullptr;
MP_RETURN_IF_ERROR(scene_cropper_->CropFrames(
scene_summary, scene_frame_timestamps_.size(), scene_frames_or_empty_,
focus_point_frames, prior_focus_point_frames_, top_static_border_size,
bottom_static_border_size, &crop_from_locations, cropped_frames_ptr));
scene_summary, scene_frame_timestamps_, is_key_frames_,
scene_frames_or_empty_, focus_point_frames, prior_focus_point_frames_,
top_static_border_size, bottom_static_border_size, continue_last_scene_,
&crop_from_locations, cropped_frames_ptr));
// Formats and outputs cropped frames.
bool apply_padding = false;
float vertical_fill_percent;
std::vector<cv::Rect> render_to_locations;
std::vector<cv::Scalar> padding_colors;
if (should_perform_frame_cropping_) {
MP_RETURN_IF_ERROR(FormatAndOutputCroppedFrames(
cropped_frames, &render_to_locations, &apply_padding, &padding_colors,
&vertical_fill_percent, cc));
}
MP_RETURN_IF_ERROR(FormatAndOutputCroppedFrames(
scene_summary.crop_window_width(), scene_summary.crop_window_height(),
scene_frame_timestamps_.size(), &render_to_locations, &apply_padding,
&padding_colors, &vertical_fill_percent, cropped_frames_ptr, cc));
// Caches prior FocusPointFrames if this was not the end of a scene.
prior_focus_point_frames_.clear();
if (!is_end_of_scene) {
const int start =
std::max(0, static_cast<int>(scene_frame_timestamps_.size()) -
options_.prior_frame_buffer_size());
options_.camera_motion_options()
.polynomial_path_solver()
.prior_frame_buffer_size());
for (int i = start; i < num_key_frames; ++i) {
prior_focus_point_frames_.push_back(focus_point_frames[i]);
}
@@ -512,6 +606,7 @@ void SceneCroppingCalculator::FilterKeyFrameInfo() {
// Optionally outputs visualization frames.
MP_RETURN_IF_ERROR(OutputVizFrames(key_frame_crop_results, focus_point_frames,
crop_from_locations,
scene_summary.crop_window_width(),
scene_summary.crop_window_height(), cc));
@@ -563,18 +658,13 @@ void SceneCroppingCalculator::FilterKeyFrameInfo() {
}
::mediapipe::Status SceneCroppingCalculator::FormatAndOutputCroppedFrames(
const std::vector<cv::Mat>& cropped_frames,
const int crop_width, const int crop_height, const int num_frames,
std::vector<cv::Rect>* render_to_locations, bool* apply_padding,
std::vector<cv::Scalar>* padding_colors, float* vertical_fill_percent,
CalculatorContext* cc) {
const std::vector<cv::Mat>* cropped_frames_ptr, CalculatorContext* cc) {
RET_CHECK(apply_padding) << "Has padding boolean is null.";
if (cropped_frames.empty()) {
return ::mediapipe::OkStatus();
}
// Computes scaling factor and decides if padding is needed.
const int crop_width = cropped_frames.front().cols;
const int crop_height = cropped_frames.front().rows;
VLOG(1) << "crop_width = " << crop_width << " crop_height = " << crop_height;
const double scaling =
std::max(static_cast<double>(target_width_) / crop_width,
@@ -602,7 +692,6 @@ void SceneCroppingCalculator::FilterKeyFrameInfo() {
// Compute the "render to" location. This is where the rect taken from the
// input video gets pasted on the output frame. For use with external
// rendering solutions.
const int num_frames = cropped_frames.size();
for (int i = 0; i < num_frames; i++) {
if (*apply_padding) {
render_to_locations->push_back(padder_->ComputeOutputLocation());
@@ -612,26 +701,11 @@ void SceneCroppingCalculator::FilterKeyFrameInfo() {
}
}
// Resizes cropped frames, pads frames, and output frames.
cv::Scalar* background_color = nullptr;
cv::Scalar interpolated_color;
// Compute padding colors.
for (int i = 0; i < num_frames; ++i) {
// Set default padding color to white.
cv::Scalar padding_color_to_add = cv::Scalar(255, 255, 255);
const int64 time_ms = scene_frame_timestamps_[i];
const Timestamp timestamp(time_ms);
auto scaled_frame = absl::make_unique<ImageFrame>(
frame_format_, scaled_width, scaled_height);
auto destination = formats::MatView(scaled_frame.get());
if (scaled_width == crop_width && scaled_height == crop_height) {
cropped_frames[i].copyTo(destination);
} else {
// cubic is better quality for upscaling and area is good for downscaling
const int interpolation_method =
scaling > 1 ? cv::INTER_CUBIC : cv::INTER_AREA;
cv::resize(cropped_frames[i], destination, destination.size(), 0, 0,
interpolation_method);
}
if (*apply_padding) {
if (has_solid_background_) {
double lab[3];
@@ -648,11 +722,39 @@ void SceneCroppingCalculator::FilterKeyFrameInfo() {
k[0] = k[0] < 0.0 ? 0.0 : k[0] > 255.0 ? 255.0 : k[0];
k[1] = k[1] < 0.0 ? 0.0 : k[1] > 255.0 ? 255.0 : k[1];
k[2] = k[2] < 0.0 ? 0.0 : k[2] > 255.0 ? 255.0 : k[2];
interpolated_color =
cv::Scalar interpolated_color =
cv::Scalar(std::round(k[0]), std::round(k[1]), std::round(k[2]));
background_color = &interpolated_color;
padding_color_to_add = interpolated_color;
}
}
padding_colors->push_back(padding_color_to_add);
}
if (!cropped_frames_ptr) {
return ::mediapipe::OkStatus();
}
// Resizes cropped frames, pads frames, and output frames.
for (int i = 0; i < num_frames; ++i) {
const int64 time_ms = scene_frame_timestamps_[i];
const Timestamp timestamp(time_ms);
auto scaled_frame = absl::make_unique<ImageFrame>(
frame_format_, scaled_width, scaled_height);
auto destination = formats::MatView(scaled_frame.get());
if (scaled_width == crop_width && scaled_height == crop_height) {
cropped_frames_ptr->at(i).copyTo(destination);
} else {
// cubic is better quality for upscaling and area is good for
// downscaling
const int interpolation_method =
scaling > 1 ? cv::INTER_CUBIC : cv::INTER_AREA;
cv::resize(cropped_frames_ptr->at(i), destination, destination.size(), 0,
0, interpolation_method);
}
if (*apply_padding) {
cv::Scalar* background_color = nullptr;
if (has_solid_background_) {
background_color = &padding_colors->at(i);
}
auto padded_frame = absl::make_unique<ImageFrame>();
MP_RETURN_IF_ERROR(padder_->Process(
*scaled_frame, background_contrast_,
@@ -670,7 +772,6 @@ void SceneCroppingCalculator::FilterKeyFrameInfo() {
.Tag(kOutputCroppedFrames)
.Add(scaled_frame.release(), timestamp);
}
padding_colors->push_back(padding_color_to_add);
}
return ::mediapipe::OkStatus();
}
@@ -678,6 +779,7 @@ void SceneCroppingCalculator::FilterKeyFrameInfo() {
mediapipe::Status SceneCroppingCalculator::OutputVizFrames(
const std::vector<KeyFrameCropResult>& key_frame_crop_results,
const std::vector<FocusPointFrame>& focus_point_frames,
const std::vector<cv::Rect>& crop_from_locations,
const int crop_window_width, const int crop_window_height,
CalculatorContext* cc) const {
if (cc->Outputs().HasTag(kOutputKeyFrameCropViz)) {
@@ -703,6 +805,17 @@ mediapipe::Status SceneCroppingCalculator::OutputVizFrames(
.Add(viz_frames[i].release(), Timestamp(scene_frame_timestamps_[i]));
}
}
if (cc->Outputs().HasTag(kOutputFramingAndDetections)) {
std::vector<std::unique_ptr<ImageFrame>> viz_frames;
MP_RETURN_IF_ERROR(DrawDetectionAndFramingWindow(
raw_scene_frames_or_empty_, crop_from_locations, frame_format_,
options_.viz_overlay_opacity(), &viz_frames));
for (int i = 0; i < raw_scene_frames_or_empty_.size(); ++i) {
cc->Outputs()
.Tag(kOutputFramingAndDetections)
.Add(viz_frames[i].release(), Timestamp(scene_frame_timestamps_[i]));
}
}
return ::mediapipe::OkStatus();
}
mediapipe/examples/desktop/autoflip/calculators/scene_cropping_calculator.h
+37 -19
View File
@@ -145,9 +145,13 @@ class SceneCroppingCalculator : public CalculatorBase {
// Removes any static borders from the scene frames before cropping. The
// arguments |top_border_size| and |bottom_border_size| report the size of the
// removed borders.
::mediapipe::Status RemoveStaticBorders(int* top_border_size,
::mediapipe::Status RemoveStaticBorders(CalculatorContext* cc,
int* top_border_size,
int* bottom_border_size);
// Sets up autoflip after first frame is received and input size is known.
::mediapipe::Status InitializeSceneCroppingCalculator(
::mediapipe::CalculatorContext* cc);
// Initializes a FrameCropRegionComputer given input and target frame sizes.
::mediapipe::Status InitializeFrameCropRegionComputer();
@@ -164,20 +168,26 @@ class SceneCroppingCalculator : public CalculatorBase {
::mediapipe::Status ProcessScene(const bool is_end_of_scene,
CalculatorContext* cc);
// Formats and outputs the cropped frames. Scales them to be at least as big
// as the target size. If the aspect ratio is different, applies padding. Uses
// solid background from static features if possible, otherwise uses blurred
// background. Sets apply_padding to true if the scene is padded.
// Formats and outputs the cropped frames passed in through
// |cropped_frames_ptr|. Scales them to be at least as big as the target
// size. If the aspect ratio is different, applies padding. Uses solid
// background from static features if possible, otherwise uses blurred
// background. Sets |apply_padding| to true if the scene is padded. Set
// |cropped_frames_ptr| to nullptr, to bypass the actual output of the
// cropped frames. This is useful when the calculator is only used for
// computing the cropping metadata rather than doing the actual cropping
// operation.
::mediapipe::Status FormatAndOutputCroppedFrames(
const std::vector<cv::Mat>& cropped_frames,
const int crop_width, const int crop_height, const int num_frames,
std::vector<cv::Rect>* render_to_locations, bool* apply_padding,
std::vector<cv::Scalar>* padding_colors, float* vertical_fill_percent,
CalculatorContext* cc);
const std::vector<cv::Mat>* cropped_frames_ptr, CalculatorContext* cc);
// Draws and outputs visualization frames if those streams are present.
::mediapipe::Status OutputVizFrames(
const std::vector<KeyFrameCropResult>& key_frame_crop_results,
const std::vector<FocusPointFrame>& focus_point_frames,
const std::vector<cv::Rect>& crop_from_locations,
const int crop_window_width, const int crop_window_height,
CalculatorContext* cc) const;
@@ -201,16 +211,21 @@ class SceneCroppingCalculator : public CalculatorBase {
// Calculator options.
SceneCroppingCalculatorOptions options_;
// Buffered KeyFrameInfos for the current scene (size = number of key frames).
// Buffered KeyFrameInfos for the current scene (size = number of key
// frames).
std::vector<KeyFrameInfo> key_frame_infos_;
// Buffered frames, timestamps, and indicators for key frames in the current
// scene (size = number of input video frames).
// Note: scene_frames_or_empty_ may be empty if the actual cropping operation
// of frames is turned off, e.g. when |should_perform_frame_cropping_| is
// false, so rely on scene_frame_timestamps_.size() to query the number of
// accumulated timestamps rather than scene_frames_or_empty_.size().
// Note: scene_frames_or_empty_ may be empty if the actual cropping
// operation of frames is turned off, e.g. when
// |should_perform_frame_cropping_| is false, so rely on
// scene_frame_timestamps_.size() to query the number of accumulated
// timestamps rather than scene_frames_or_empty_.size().
// TODO: all of the following vectors are expected to be the same
// size. Add to struct and store together in one vector.
std::vector<cv::Mat> scene_frames_or_empty_;
std::vector<cv::Mat> raw_scene_frames_or_empty_;
std::vector<int64> scene_frame_timestamps_;
std::vector<bool> is_key_frames_;
@@ -221,6 +236,9 @@ class SceneCroppingCalculator : public CalculatorBase {
// Stored FocusPointFrames from prior scene when there was no actual scene
// change (due to forced flush when buffer is full).
std::vector<FocusPointFrame> prior_focus_point_frames_;
// Indicates if this scene is a continuation of the last scene (due to
// forced flush when buffer is full).
bool continue_last_scene_ = false;
// KeyFrameCropOptions used by the FrameCropRegionComputer.
KeyFrameCropOptions key_frame_crop_options_;
@@ -242,8 +260,8 @@ class SceneCroppingCalculator : public CalculatorBase {
std::vector<StaticFeatures> static_features_;
std::vector<int64> static_features_timestamps_;
bool has_solid_background_ = false;
// CIELAB yields more natural color transitions than RGB and HSV: RGB tends to
// produce darker in-between colors and HSV can introduce new hues. See
// CIELAB yields more natural color transitions than RGB and HSV: RGB tends
// to produce darker in-between colors and HSV can introduce new hues. See
// https://howaboutanorange.com/blog/2011/08/10/color_interpolation/ for
// visual comparisons of color transition in different spaces.
PiecewiseLinearFunction background_color_l_function_; // CIELAB - l
@@ -264,11 +282,11 @@ class SceneCroppingCalculator : public CalculatorBase {
std::unique_ptr<std::vector<ExternalRenderFrame>> external_render_list_;
// Determines whether to perform real cropping on input frames. This flag is
// useful when the user only needs to compute cropping windows, in which case
// setting this flag to false can avoid buffering as well as cropping frames.
// This can significantly reduce memory usage and speed up processing. Some
// debugging visualization inevitably will be disabled because of this flag
// too.
// useful when the user only needs to compute cropping windows, in which
// case setting this flag to false can avoid buffering as well as cropping
// frames. This can significantly reduce memory usage and speed up
// processing. Some debugging visualization inevitably will be disabled
// because of this flag too.
bool should_perform_frame_cropping_ = false;
};
} // namespace autoflip
mediapipe/examples/desktop/autoflip/calculators/scene_cropping_calculator.proto
+6 -1
View File
@@ -44,6 +44,8 @@ message SceneCroppingCalculatorOptions {
// Used on conjuntion with external_aspect_ratio, create the largest sized
// output without upscaling the video.
MAXIMIZE_TARGET_DIMENSION = 4;
// Uses original dimensions to calculate aspect ratio.
KEEP_ORIGINAL_DIMENSION = 5;
}
optional TargetSizeType target_size_type = 3 [default = USE_TARGET_DIMENSION];
@@ -53,7 +55,10 @@ message SceneCroppingCalculatorOptions {
// Number of frames from prior buffer to be used to smooth out camera
// trajectory when it was a forced flush.
optional int32 prior_frame_buffer_size = 5 [default = 30];
optional int32 prior_frame_buffer_size = 5 [default = 30, deprecated = true];
// Set camera motion type along with parameters. Must select between the two
// provided options.
optional CameraMotionOptions camera_motion_options = 14;
// Options for computing key frame crop regions using the
// FrameCropRegionComputer.
mediapipe/examples/desktop/autoflip/calculators/scene_cropping_calculator_test.cc
+260 -23
View File
@@ -94,6 +94,7 @@ constexpr char kDebugConfig[] = R"(
output_stream: "CROPPED_FRAMES:cropped_frames"
output_stream: "KEY_FRAME_CROP_REGION_VIZ_FRAMES:key_frame_crop_viz_frames"
output_stream: "SALIENT_POINT_FRAME_VIZ_FRAMES:salient_point_viz_frames"
output_stream: "FRAMING_DETECTIONS_VIZ_FRAMES:framing_viz_frames"
output_stream: "CROPPING_SUMMARY:cropping_summaries"
output_stream: "EXTERNAL_RENDERING_PER_FRAME:external_rendering_per_frame"
output_stream: "EXTERNAL_RENDERING_FULL_VID:external_rendering_full_vid"
@@ -104,6 +105,39 @@ constexpr char kDebugConfig[] = R"(
}
})";
constexpr char kExternalRenderConfig[] = R"(
calculator: "SceneCroppingCalculator"
input_stream: "VIDEO_FRAMES:camera_frames_org"
input_stream: "KEY_FRAMES:down_sampled_frames"
input_stream: "DETECTION_FEATURES:salient_regions"
input_stream: "STATIC_FEATURES:border_features"
input_stream: "SHOT_BOUNDARIES:shot_boundary_frames"
output_stream: "EXTERNAL_RENDERING_PER_FRAME:external_rendering_per_frame"
output_stream: "EXTERNAL_RENDERING_FULL_VID:external_rendering_full_vid"
options: {
[mediapipe.autoflip.SceneCroppingCalculatorOptions.ext]: {
target_width: $0
target_height: $1
}
})";
constexpr char kExternalRenderConfigNoVideo[] = R"(
calculator: "SceneCroppingCalculator"
input_stream: "VIDEO_SIZE:camera_size"
input_stream: "DETECTION_FEATURES:salient_regions"
input_stream: "STATIC_FEATURES:border_features"
input_stream: "SHOT_BOUNDARIES:shot_boundary_frames"
output_stream: "EXTERNAL_RENDERING_PER_FRAME:external_rendering_per_frame"
output_stream: "EXTERNAL_RENDERING_FULL_VID:external_rendering_full_vid"
options: {
[mediapipe.autoflip.SceneCroppingCalculatorOptions.ext]: {
target_width: $0
target_height: $1
video_features_width: $2
video_features_height: $3
}
})";
constexpr int kInputFrameWidth = 1280;
constexpr int kInputFrameHeight = 720;
@@ -172,6 +206,22 @@ std::unique_ptr<DetectionSet> MakeDetections(const int num_detections,
return detections;
}
// Makes a detection set given number of detections. Each detection has randomly
// generated regions within given width and height with random score in [0, 1],
// and is randomly set to be required or non-required.
std::unique_ptr<DetectionSet> MakeCenterDetection(const int width,
const int height) {
auto detections = absl::make_unique<DetectionSet>();
auto* region = detections->add_detections();
auto* location = region->mutable_location();
location->set_x(width / 2 - 5);
location->set_width(width / 2 + 10);
location->set_y(height / 2 - 5);
location->set_height(height);
region->set_score(1);
return detections;
}
// Makes an image frame of solid color given color, width, and height.
std::unique_ptr<ImageFrame> MakeImageFrameFromColor(const cv::Scalar& color,
const int width,
@@ -188,7 +238,7 @@ std::unique_ptr<ImageFrame> MakeImageFrameFromColor(const cv::Scalar& color,
// and kMaxNumDetections. Optionally add a key image frame of random solid color
// and given size.
void AddKeyFrameFeatures(const int64 time_ms, const int key_frame_width,
const int key_frame_height,
const int key_frame_height, bool randomize,
CalculatorRunner::StreamContentsSet* inputs) {
Timestamp timestamp(time_ms);
if (inputs->HasTag("KEY_FRAMES")) {
@@ -197,13 +247,18 @@ void AddKeyFrameFeatures(const int64 time_ms, const int key_frame_width,
inputs->Tag("KEY_FRAMES")
.packets.push_back(Adopt(key_frame.release()).At(timestamp));
}
const int num_detections = std::uniform_int_distribution<int>(
kMinNumDetections, kMaxNumDetections)(GetGen());
auto detections =
MakeDetections(num_detections, key_frame_width, key_frame_height);
inputs->Tag("DETECTION_FEATURES")
.packets.push_back(Adopt(detections.release()).At(timestamp));
if (randomize) {
const int num_detections = std::uniform_int_distribution<int>(
kMinNumDetections, kMaxNumDetections)(GetGen());
auto detections =
MakeDetections(num_detections, key_frame_width, key_frame_height);
inputs->Tag("DETECTION_FEATURES")
.packets.push_back(Adopt(detections.release()).At(timestamp));
} else {
auto detections = MakeCenterDetection(key_frame_width, key_frame_height);
inputs->Tag("DETECTION_FEATURES")
.packets.push_back(Adopt(detections.release()).At(timestamp));
}
}
// Adds a scene given number of frames to the input stream. Spaces frame at the
@@ -212,19 +267,31 @@ void AddKeyFrameFeatures(const int64 time_ms, const int key_frame_width,
void AddScene(const int start_frame_index, const int num_scene_frames,
const int frame_width, const int frame_height,
const int key_frame_width, const int key_frame_height,
const int DownSampleRate,
CalculatorRunner::StreamContentsSet* inputs) {
int64 time_ms = start_frame_index * kTimestampDiff;
for (int i = 0; i < num_scene_frames; ++i) {
Timestamp timestamp(time_ms);
auto frame =
MakeImageFrameFromColor(GetRandomColor(), frame_width, frame_height);
inputs->Tag("VIDEO_FRAMES")
.packets.push_back(Adopt(frame.release()).At(timestamp));
if (inputs->HasTag("VIDEO_FRAMES")) {
auto frame =
MakeImageFrameFromColor(GetRandomColor(), frame_width, frame_height);
inputs->Tag("VIDEO_FRAMES")
.packets.push_back(Adopt(frame.release()).At(timestamp));
} else {
auto input_size =
::absl::make_unique<std::pair<int, int>>(frame_width, frame_height);
inputs->Tag("VIDEO_SIZE")
.packets.push_back(Adopt(input_size.release()).At(timestamp));
}
auto static_features = absl::make_unique<StaticFeatures>();
inputs->Tag("STATIC_FEATURES")
.packets.push_back(Adopt(static_features.release()).At(timestamp));
if (i % kDownSampleRate == 0) { // is a key frame
AddKeyFrameFeatures(time_ms, key_frame_width, key_frame_height, inputs);
if (DownSampleRate == 1) {
AddKeyFrameFeatures(time_ms, key_frame_width, key_frame_height, false,
inputs);
} else if (i % DownSampleRate == 0) { // is a key frame
AddKeyFrameFeatures(time_ms, key_frame_width, key_frame_height, true,
inputs);
}
if (i == num_scene_frames - 1) { // adds shot boundary
inputs->Tag("SHOT_BOUNDARIES")
@@ -294,7 +361,7 @@ TEST(SceneCroppingCalculatorTest, HandlesNoKeyFrames) {
absl::Substitute(kNoKeyFrameConfig, kTargetWidth, kTargetHeight));
auto runner = absl::make_unique<CalculatorRunner>(config);
AddScene(0, kSceneSize, kInputFrameWidth, kInputFrameHeight, kKeyFrameWidth,
kKeyFrameHeight, runner->MutableInputs());
kKeyFrameHeight, kDownSampleRate, runner->MutableInputs());
MP_EXPECT_OK(runner->Run());
CheckCroppedFrames(*runner, kSceneSize, kTargetWidth, kTargetHeight);
}
@@ -308,7 +375,8 @@ TEST(SceneCroppingCalculatorTest, HandlesLongScene) {
kPriorFrameBufferSize));
auto runner = absl::make_unique<CalculatorRunner>(config);
AddScene(0, 2 * kMaxSceneSize, kInputFrameWidth, kInputFrameHeight,
kKeyFrameWidth, kKeyFrameHeight, runner->MutableInputs());
kKeyFrameWidth, kKeyFrameHeight, kDownSampleRate,
runner->MutableInputs());
MP_EXPECT_OK(runner->Run());
CheckCroppedFrames(*runner, 2 * kMaxSceneSize, kTargetWidth, kTargetHeight);
}
@@ -321,7 +389,7 @@ TEST(SceneCroppingCalculatorTest, OutputsDebugStreams) {
auto runner = absl::make_unique<CalculatorRunner>(config);
const int num_frames = kSceneSize;
AddScene(0, num_frames, kInputFrameWidth, kInputFrameHeight, kKeyFrameWidth,
kKeyFrameHeight, runner->MutableInputs());
kKeyFrameHeight, kDownSampleRate, runner->MutableInputs());
MP_EXPECT_OK(runner->Run());
const auto& outputs = runner->Outputs();
@@ -330,6 +398,7 @@ TEST(SceneCroppingCalculatorTest, OutputsDebugStreams) {
EXPECT_TRUE(outputs.HasTag("CROPPING_SUMMARY"));
EXPECT_TRUE(outputs.HasTag("EXTERNAL_RENDERING_PER_FRAME"));
EXPECT_TRUE(outputs.HasTag("EXTERNAL_RENDERING_FULL_VID"));
EXPECT_TRUE(outputs.HasTag("FRAMING_DETECTIONS_VIZ_FRAMES"));
const auto& crop_region_viz_frames_outputs =
outputs.Tag("KEY_FRAME_CROP_REGION_VIZ_FRAMES").packets;
const auto& salient_point_viz_frames_outputs =
@@ -339,8 +408,11 @@ TEST(SceneCroppingCalculatorTest, OutputsDebugStreams) {
outputs.Tag("EXTERNAL_RENDERING_PER_FRAME").packets;
const auto& ext_render_full_vid =
outputs.Tag("EXTERNAL_RENDERING_FULL_VID").packets;
const auto& framing_viz_frames_output =
outputs.Tag("FRAMING_DETECTIONS_VIZ_FRAMES").packets;
EXPECT_EQ(crop_region_viz_frames_outputs.size(), num_frames);
EXPECT_EQ(salient_point_viz_frames_outputs.size(), num_frames);
EXPECT_EQ(framing_viz_frames_output.size(), num_frames);
EXPECT_EQ(summary_output.size(), 1);
EXPECT_EQ(ext_render_per_frame.size(), num_frames);
EXPECT_EQ(ext_render_full_vid.size(), 1);
@@ -387,7 +459,8 @@ TEST(SceneCroppingCalculatorTest, HandlesLandscapeTarget) {
auto runner = absl::make_unique<CalculatorRunner>(config);
for (int i = 0; i < kNumScenes; ++i) {
AddScene(i * kSceneSize, kSceneSize, input_width, input_height,
kKeyFrameWidth, kKeyFrameHeight, runner->MutableInputs());
kKeyFrameWidth, kKeyFrameHeight, kDownSampleRate,
runner->MutableInputs());
}
const int num_frames = kSceneSize * kNumScenes;
MP_EXPECT_OK(runner->Run());
@@ -404,13 +477,34 @@ TEST(SceneCroppingCalculatorTest, CropsToTargetSize) {
auto runner = absl::make_unique<CalculatorRunner>(config);
for (int i = 0; i < kNumScenes; ++i) {
AddScene(i * kSceneSize, kSceneSize, kInputFrameWidth, kInputFrameHeight,
kKeyFrameWidth, kKeyFrameHeight, runner->MutableInputs());
kKeyFrameWidth, kKeyFrameHeight, kDownSampleRate,
runner->MutableInputs());
}
const int num_frames = kSceneSize * kNumScenes;
MP_EXPECT_OK(runner->Run());
CheckCroppedFrames(*runner, num_frames, kTargetWidth, kTargetHeight);
}
// Checks that the calculator crops scene frames to input size when the target
// size type is KEEP_ORIGINAL_DIMENSION.
TEST(SceneCroppingCalculatorTest, CropsToOriginalDimension) {
// target_width and target_height are ignored
const CalculatorGraphConfig::Node config =
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(absl::Substitute(
kConfig, /*target_width*/ 2, /*target_height*/ 2,
SceneCroppingCalculatorOptions::KEEP_ORIGINAL_DIMENSION,
kMaxSceneSize, kPriorFrameBufferSize));
auto runner = absl::make_unique<CalculatorRunner>(config);
for (int i = 0; i < kNumScenes; ++i) {
AddScene(i * kSceneSize, kSceneSize, kInputFrameWidth, kInputFrameHeight,
kKeyFrameWidth, kKeyFrameHeight, kDownSampleRate,
runner->MutableInputs());
}
const int num_frames = kSceneSize * kNumScenes;
MP_EXPECT_OK(runner->Run());
CheckCroppedFrames(*runner, num_frames, kInputFrameWidth, kInputFrameHeight);
}
// Checks that the calculator keeps original height if the target size type is
// set to KEEP_ORIGINAL_HEIGHT.
TEST(SceneCroppingCalculatorTest, KeepsOriginalHeight) {
@@ -427,7 +521,8 @@ TEST(SceneCroppingCalculatorTest, KeepsOriginalHeight) {
kPriorFrameBufferSize));
auto runner = absl::make_unique<CalculatorRunner>(config);
AddScene(0, kMaxSceneSize, kInputFrameWidth, kInputFrameHeight,
kKeyFrameWidth, kKeyFrameHeight, runner->MutableInputs());
kKeyFrameWidth, kKeyFrameHeight, kDownSampleRate,
runner->MutableInputs());
MP_EXPECT_OK(runner->Run());
CheckCroppedFrames(*runner, kMaxSceneSize, target_width, target_height);
}
@@ -448,7 +543,8 @@ TEST(SceneCroppingCalculatorTest, KeepsOriginalWidth) {
kPriorFrameBufferSize));
auto runner = absl::make_unique<CalculatorRunner>(config);
AddScene(0, kMaxSceneSize, kInputFrameWidth, kInputFrameHeight,
kKeyFrameWidth, kKeyFrameHeight, runner->MutableInputs());
kKeyFrameWidth, kKeyFrameHeight, kDownSampleRate,
runner->MutableInputs());
MP_EXPECT_OK(runner->Run());
CheckCroppedFrames(*runner, kMaxSceneSize, target_width, target_height);
}
@@ -461,7 +557,8 @@ TEST(SceneCroppingCalculatorTest, RejectsOddTargetSize) {
kMaxSceneSize, kPriorFrameBufferSize));
auto runner = absl::make_unique<CalculatorRunner>(config);
AddScene(0, kMaxSceneSize, kInputFrameWidth, kInputFrameHeight,
kKeyFrameWidth, kKeyFrameHeight, runner->MutableInputs());
kKeyFrameWidth, kKeyFrameHeight, kDownSampleRate,
runner->MutableInputs());
const auto status = runner->Run();
EXPECT_FALSE(status.ok());
EXPECT_THAT(status.ToString(), HasSubstr("Target width cannot be odd"));
@@ -498,7 +595,7 @@ TEST(SceneCroppingCalculatorTest, ProducesEvenFrameSize) {
kMaxSceneSize, kPriorFrameBufferSize));
auto runner = absl::make_unique<CalculatorRunner>(config);
AddScene(0, 1, frame_width, frame_height, kKeyFrameWidth,
kKeyFrameHeight, runner->MutableInputs());
kKeyFrameHeight, kDownSampleRate, runner->MutableInputs());
MP_EXPECT_OK(runner->Run());
const auto& output_frame = runner->Outputs()
.Tag("CROPPED_FRAMES")
@@ -665,6 +762,146 @@ TEST(SceneCroppingCalculatorTest, RemovesStaticBorders) {
}
}
}
// Checks external render message with default poly path solver.
TEST(SceneCroppingCalculatorTest, OutputsCropMessagePolyPath) {
const CalculatorGraphConfig::Node config =
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(
absl::Substitute(kExternalRenderConfig, kTargetWidth, kTargetHeight));
auto runner = absl::make_unique<CalculatorRunner>(config);
const int num_frames = kSceneSize;
AddScene(0, num_frames, kInputFrameWidth, kInputFrameHeight, kKeyFrameWidth,
kKeyFrameHeight, 1, runner->MutableInputs());
MP_EXPECT_OK(runner->Run());
const auto& outputs = runner->Outputs();
const auto& ext_render_per_frame =
outputs.Tag("EXTERNAL_RENDERING_PER_FRAME").packets;
EXPECT_EQ(ext_render_per_frame.size(), num_frames);
for (int i = 0; i < num_frames - 1; ++i) {
const auto& ext_render_message =
ext_render_per_frame[i].Get<ExternalRenderFrame>();
EXPECT_EQ(ext_render_message.timestamp_us(), i * 20000);
EXPECT_EQ(ext_render_message.crop_from_location().x(), 725);
EXPECT_EQ(ext_render_message.crop_from_location().y(), 0);
EXPECT_EQ(ext_render_message.crop_from_location().width(), 461);
EXPECT_EQ(ext_render_message.crop_from_location().height(), 720);
EXPECT_EQ(ext_render_message.render_to_location().x(), 0);
EXPECT_EQ(ext_render_message.render_to_location().y(), 0);
EXPECT_EQ(ext_render_message.render_to_location().width(), 720);
EXPECT_EQ(ext_render_message.render_to_location().height(), 1124);
}
}
// Checks external render message with kinematic path solver.
TEST(SceneCroppingCalculatorTest, OutputsCropMessageKinematicPath) {
CalculatorGraphConfig::Node config =
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(
absl::Substitute(kDebugConfig, kTargetWidth, kTargetHeight));
auto* options = config.mutable_options()->MutableExtension(
SceneCroppingCalculatorOptions::ext);
auto* kinematic_options =
options->mutable_camera_motion_options()->mutable_kinematic_options();
kinematic_options->set_max_velocity(200);
auto runner = absl::make_unique<CalculatorRunner>(config);
const int num_frames = kSceneSize;
AddScene(0, num_frames, kInputFrameWidth, kInputFrameHeight, kKeyFrameWidth,
kKeyFrameHeight, 1, runner->MutableInputs());
MP_EXPECT_OK(runner->Run());
const auto& outputs = runner->Outputs();
const auto& ext_render_per_frame =
outputs.Tag("EXTERNAL_RENDERING_PER_FRAME").packets;
EXPECT_EQ(ext_render_per_frame.size(), num_frames);
for (int i = 0; i < num_frames - 1; ++i) {
const auto& ext_render_message =
ext_render_per_frame[i].Get<ExternalRenderFrame>();
EXPECT_EQ(ext_render_message.timestamp_us(), i * 20000);
EXPECT_EQ(ext_render_message.crop_from_location().x(), 725);
EXPECT_EQ(ext_render_message.crop_from_location().y(), 0);
EXPECT_EQ(ext_render_message.crop_from_location().width(), 461);
EXPECT_EQ(ext_render_message.crop_from_location().height(), 720);
EXPECT_EQ(ext_render_message.render_to_location().x(), 0);
EXPECT_EQ(ext_render_message.render_to_location().y(), 0);
EXPECT_EQ(ext_render_message.render_to_location().width(), 720);
EXPECT_EQ(ext_render_message.render_to_location().height(), 1124);
}
}
// Checks external render message with default poly path solver without video
// input.
TEST(SceneCroppingCalculatorTest, OutputsCropMessagePolyPathNoVideo) {
const CalculatorGraphConfig::Node config =
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(
absl::Substitute(kExternalRenderConfigNoVideo, kTargetWidth,
kTargetHeight, kKeyFrameWidth, kKeyFrameHeight));
auto runner = absl::make_unique<CalculatorRunner>(config);
const int num_frames = kSceneSize;
AddScene(0, num_frames, kInputFrameWidth, kInputFrameHeight, kKeyFrameWidth,
kKeyFrameHeight, 1, runner->MutableInputs());
MP_EXPECT_OK(runner->Run());
const auto& outputs = runner->Outputs();
const auto& ext_render_per_frame =
outputs.Tag("EXTERNAL_RENDERING_PER_FRAME").packets;
EXPECT_EQ(ext_render_per_frame.size(), num_frames);
for (int i = 0; i < num_frames - 1; ++i) {
const auto& ext_render_message =
ext_render_per_frame[i].Get<ExternalRenderFrame>();
EXPECT_EQ(ext_render_message.timestamp_us(), i * 20000);
EXPECT_EQ(ext_render_message.crop_from_location().x(), 725);
EXPECT_EQ(ext_render_message.crop_from_location().y(), 0);
EXPECT_EQ(ext_render_message.crop_from_location().width(), 461);
EXPECT_EQ(ext_render_message.crop_from_location().height(), 720);
EXPECT_EQ(ext_render_message.render_to_location().x(), 0);
EXPECT_EQ(ext_render_message.render_to_location().y(), 0);
EXPECT_EQ(ext_render_message.render_to_location().width(), 720);
EXPECT_EQ(ext_render_message.render_to_location().height(), 1124);
}
}
// Checks external render message with kinematic path solver without video
// input.
TEST(SceneCroppingCalculatorTest, OutputsCropMessageKinematicPathNoVideo) {
CalculatorGraphConfig::Node config =
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(
absl::Substitute(kExternalRenderConfigNoVideo, kTargetWidth,
kTargetHeight, kKeyFrameWidth, kKeyFrameHeight));
auto* options = config.mutable_options()->MutableExtension(
SceneCroppingCalculatorOptions::ext);
auto* kinematic_options =
options->mutable_camera_motion_options()->mutable_kinematic_options();
kinematic_options->set_max_velocity(2.0);
auto runner = absl::make_unique<CalculatorRunner>(config);
const int num_frames = kSceneSize;
AddScene(0, num_frames, kInputFrameWidth, kInputFrameHeight, kKeyFrameWidth,
kKeyFrameHeight, 1, runner->MutableInputs());
MP_EXPECT_OK(runner->Run());
const auto& outputs = runner->Outputs();
const auto& ext_render_per_frame =
outputs.Tag("EXTERNAL_RENDERING_PER_FRAME").packets;
EXPECT_EQ(ext_render_per_frame.size(), num_frames);
for (int i = 0; i < num_frames - 1; ++i) {
const auto& ext_render_message =
ext_render_per_frame[i].Get<ExternalRenderFrame>();
EXPECT_EQ(ext_render_message.timestamp_us(), i * 20000);
EXPECT_EQ(ext_render_message.crop_from_location().x(), 725);
EXPECT_EQ(ext_render_message.crop_from_location().y(), 0);
EXPECT_EQ(ext_render_message.crop_from_location().width(), 461);
EXPECT_EQ(ext_render_message.crop_from_location().height(), 720);
EXPECT_EQ(ext_render_message.render_to_location().x(), 0);
EXPECT_EQ(ext_render_message.render_to_location().y(), 0);
EXPECT_EQ(ext_render_message.render_to_location().width(), 720);
EXPECT_EQ(ext_render_message.render_to_location().height(), 1124);
}
}
} // namespace
} // namespace autoflip
} // namespace mediapipe
mediapipe/examples/desktop/autoflip/calculators/signal_fusing_calculator.cc
+108 -20
View File
@@ -30,6 +30,10 @@ using mediapipe::autoflip::DetectionSet;
using mediapipe::autoflip::SalientRegion;
using mediapipe::autoflip::SignalType;
constexpr char kIsShotBoundaryTag[] = "IS_SHOT_BOUNDARY";
constexpr char kSignalInputsTag[] = "SIGNAL";
constexpr char kOutputTag[] = "OUTPUT";
namespace mediapipe {
namespace autoflip {
@@ -43,16 +47,16 @@ struct Frame {
mediapipe::Timestamp time;
};
// This calculator takes one scene change signal and an arbitrary number of
// detection signals and outputs a single list of detections. The scores for
// the detections can be re-normalized using the options proto. Additionally,
// if a detection has a consistent tracking id during a scene the score for that
// detection is averaged over the whole scene.
// This calculator takes one scene change signal (optional, see below) and an
// arbitrary number of detection signals and outputs a single list of
// detections. The scores for the detections can be re-normalized using the
// options proto. Additionally, if a detection has a consistent tracking id
// during a scene the score for that detection is averaged over the whole scene.
//
// Example:
// Example (ordered interface):
// node {
// calculator: "SignalFusingCalculator"
// input_stream: "scene_change"
// input_stream: "scene_change" (required for ordered interface)
// input_stream: "detection_faces"
// input_stream: "detection_custom_text"
// output_stream: "salient_region"
@@ -71,9 +75,33 @@ struct Frame {
// }
// }
// }
//
// Example (tag interface):
// node {
// calculator: "SignalFusingCalculator"
// input_stream: "IS_SHOT_BOUNDARY:scene_change" (optional)
// input_stream: "SIGNAL:0:detection_faces"
// input_stream: "SIGNAL:1:detection_custom_text"
// output_stream: "OUTPUT:salient_region"
// options:{
// [mediapipe.autoflip.SignalFusingCalculatorOptions.ext]:{
// signal_settings{
// type: {standard: FACE}
// min_score: 0.5
// max_score: 0.6
// }
// signal_settings{
// type: {custom: "custom_text"}
// min_score: 0.9
// max_score: 1.0
// }
// }
// }
// }
class SignalFusingCalculator : public mediapipe::CalculatorBase {
public:
SignalFusingCalculator() {}
SignalFusingCalculator()
: tag_input_interface_(false), process_by_scene_(true) {}
SignalFusingCalculator(const SignalFusingCalculator&) = delete;
SignalFusingCalculator& operator=(const SignalFusingCalculator&) = delete;
@@ -84,9 +112,12 @@ class SignalFusingCalculator : public mediapipe::CalculatorBase {
private:
mediapipe::Status ProcessScene(mediapipe::CalculatorContext* cc);
std::vector<Packet> GetSignalPackets(mediapipe::CalculatorContext* cc);
SignalFusingCalculatorOptions options_;
std::map<std::string, SignalSettings> settings_by_type_;
std::vector<Frame> scene_frames_;
bool tag_input_interface_;
bool process_by_scene_;
};
REGISTER_CALCULATOR(SignalFusingCalculator);
@@ -104,7 +135,23 @@ std::string CreateKey(const InputSignal& detection) {
std::string id = id_source + ":" + id_signal;
return id;
}
void SetupTagInput(mediapipe::CalculatorContract* cc) {
if (cc->Inputs().HasTag(kIsShotBoundaryTag)) {
cc->Inputs().Tag(kIsShotBoundaryTag).Set<bool>();
}
for (int i = 0; i < cc->Inputs().NumEntries(kSignalInputsTag); i++) {
cc->Inputs().Get(kSignalInputsTag, i).Set<autoflip::DetectionSet>();
}
cc->Outputs().Tag(kOutputTag).Set<autoflip::DetectionSet>();
}
void SetupOrderedInput(mediapipe::CalculatorContract* cc) {
cc->Inputs().Index(0).Set<bool>();
for (int i = 1; i < cc->Inputs().NumEntries(); ++i) {
cc->Inputs().Index(i).Set<autoflip::DetectionSet>();
}
cc->Outputs().Index(0).Set<autoflip::DetectionSet>();
}
} // namespace
mediapipe::Status SignalFusingCalculator::Open(
@@ -113,6 +160,12 @@ mediapipe::Status SignalFusingCalculator::Open(
for (const auto& setting : options_.signal_settings()) {
settings_by_type_[CreateSettingsKey(setting.type())] = setting;
}
if (cc->Inputs().HasTag(kSignalInputsTag)) {
tag_input_interface_ = true;
if (!cc->Inputs().HasTag(kIsShotBoundaryTag)) {
process_by_scene_ = false;
}
}
return ::mediapipe::OkStatus();
}
@@ -144,14 +197,12 @@ mediapipe::Status SignalFusingCalculator::ProcessScene(
}
}
}
// Average scores.
for (auto iterator = multiframe_score.begin();
iterator != multiframe_score.end(); iterator++) {
multiframe_score[iterator->first] =
iterator->second / detection_count[iterator->first];
}
// Process detections.
for (const Frame& frame : scene_frames_) {
std::unique_ptr<DetectionSet> processed_detections(new DetectionSet());
@@ -173,33 +224,63 @@ mediapipe::Status SignalFusingCalculator::ProcessScene(
min_value = settings_it->second.min_score();
max_value = settings_it->second.max_score();
detection.signal.set_is_required(settings_it->second.is_required());
detection.signal.set_only_required(settings_it->second.only_required());
}
float final_score = score * (max_value - min_value) + min_value;
detection.signal.set_score(final_score);
*processed_detections->add_detections() = detection.signal;
}
cc->Outputs().Index(0).Add(processed_detections.release(), frame.time);
if (tag_input_interface_) {
cc->Outputs()
.Tag(kOutputTag)
.Add(processed_detections.release(), frame.time);
} else {
cc->Outputs().Index(0).Add(processed_detections.release(), frame.time);
}
}
return ::mediapipe::OkStatus();
}
std::vector<Packet> SignalFusingCalculator::GetSignalPackets(
mediapipe::CalculatorContext* cc) {
std::vector<Packet> signal_packets;
if (tag_input_interface_) {
for (int i = 0; i < cc->Inputs().NumEntries(kSignalInputsTag); i++) {
const Packet& packet = cc->Inputs().Get(kSignalInputsTag, i).Value();
signal_packets.push_back(packet);
}
} else {
for (int i = 1; i < cc->Inputs().NumEntries(); i++) {
const Packet& packet = cc->Inputs().Index(i).Value();
signal_packets.push_back(packet);
}
}
return signal_packets;
}
mediapipe::Status SignalFusingCalculator::Process(
mediapipe::CalculatorContext* cc) {
bool is_boundary = false;
if (!cc->Inputs().Index(0).Value().IsEmpty()) {
is_boundary = cc->Inputs().Index(0).Get<bool>();
if (process_by_scene_) {
const auto& shot_tag = (tag_input_interface_)
? cc->Inputs().Tag(kIsShotBoundaryTag)
: cc->Inputs().Index(0);
if (!shot_tag.Value().IsEmpty()) {
is_boundary = shot_tag.Get<bool>();
}
}
if (is_boundary || scene_frames_.size() > options_.max_scene_size()) {
if (is_boundary) {
MP_RETURN_IF_ERROR(ProcessScene(cc));
scene_frames_.clear();
}
Frame frame;
for (int i = 1; i < cc->Inputs().NumEntries(); ++i) {
const Packet& packet = cc->Inputs().Index(i).Value();
const auto& signal_packets = GetSignalPackets(cc);
for (int i = 0; i < signal_packets.size(); i++) {
const Packet& packet = signal_packets[i];
if (packet.IsEmpty()) {
continue;
}
@@ -214,16 +295,23 @@ mediapipe::Status SignalFusingCalculator::Process(
frame.time = cc->InputTimestamp();
scene_frames_.push_back(frame);
// Flush buffer on same input if it exceeds max_scene_size or if there is not
// shot input information.
if (scene_frames_.size() > options_.max_scene_size() || !process_by_scene_) {
MP_RETURN_IF_ERROR(ProcessScene(cc));
scene_frames_.clear();
}
return ::mediapipe::OkStatus();
}
::mediapipe::Status SignalFusingCalculator::GetContract(
mediapipe::CalculatorContract* cc) {
cc->Inputs().Index(0).Set<bool>();
for (int i = 1; i < cc->Inputs().NumEntries(); ++i) {
cc->Inputs().Index(i).Set<autoflip::DetectionSet>();
if (cc->Inputs().NumEntries(kSignalInputsTag) > 0) {
SetupTagInput(cc);
} else {
SetupOrderedInput(cc);
}
cc->Outputs().Index(0).Set<autoflip::DetectionSet>();
return ::mediapipe::OkStatus();
}
mediapipe/examples/desktop/autoflip/calculators/signal_fusing_calculator.proto
+11 -1
View File
@@ -31,7 +31,7 @@ message SignalFusingCalculatorOptions {
optional int32 max_scene_size = 2 [default = 600];
}
// Next tag: 5
// Next tag: 6
message SignalSettings {
// The type of signal these settings pertain to.
optional SignalType type = 1;
@@ -51,4 +51,14 @@ message SignalSettings {
// Is this signal required within the output cropped video? If it is it will
// be included or the video will be marked as failed to convert.
optional bool is_required = 4 [default = false];
// When used with ContentZoomingCalculator, this flag can be set indicating
// that areas outside of these salient regions can be cropped from the frame.
// When no salient regions have this flag set true, no zooming is performed.
// When one or more salient regions have this flag set true, the max zoom
// value will be used that keeps all “only_required” detections within view.
// The ContentZoomingCalculator currently supports zooming by finding the size
// of non-salient top/bottom borders regions and provides this information to
// the SceneCroppingCalculator for reframing.
optional bool only_required = 5 [default = false];
}
mediapipe/examples/desktop/autoflip/calculators/signal_fusing_calculator_test.cc
+128
View File
@@ -80,6 +80,27 @@ const char kConfigB[] = R"(
}
})";
const char kConfigC[] = R"(
calculator: "SignalFusingCalculator"
input_stream: "IS_SHOT_BOUNDARY:scene_change"
input_stream: "SIGNAL:0:detection_set_a"
input_stream: "SIGNAL:1:detection_set_b"
output_stream: "OUTPUT:salient_region"
options:{
[mediapipe.autoflip.SignalFusingCalculatorOptions.ext]:{
signal_settings{
type: {standard: FACE_FULL}
min_score: 0.5
max_score: 0.6
}
signal_settings{
type: {standard: TEXT}
min_score: 0.9
max_score: 1.0
}
}
})";
TEST(SignalFusingCalculatorTest, TwoInputNoTracking) {
auto runner = absl::make_unique<CalculatorRunner>(
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfigA));
@@ -133,6 +154,113 @@ TEST(SignalFusingCalculatorTest, TwoInputNoTracking) {
EXPECT_FLOAT_EQ(detection_set.detections(3).score(), .99);
}
TEST(SignalFusingCalculatorTest, TwoInputShotLabeledTags) {
auto runner = absl::make_unique<CalculatorRunner>(
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfigC));
auto input_shot = absl::make_unique<bool>(false);
runner->MutableInputs()
->Tag("IS_SHOT_BOUNDARY")
.packets.push_back(Adopt(input_shot.release()).At(Timestamp(0)));
auto input_face =
absl::make_unique<DetectionSet>(ParseTextProtoOrDie<DetectionSet>(
R"(
detections {
score: 0.5
signal_type: { standard: FACE_FULL }
}
detections {
score: 0.3
signal_type: { standard: FACE_FULL }
}
)"));
runner->MutableInputs()
->Get("SIGNAL", 0)
.packets.push_back(Adopt(input_face.release()).At(Timestamp(0)));
auto input_ocr =
absl::make_unique<DetectionSet>(ParseTextProtoOrDie<DetectionSet>(
R"(
detections {
score: 0.3
signal_type: { standard: TEXT }
}
detections {
score: 0.9
signal_type: { standard: TEXT }
}
)"));
runner->MutableInputs()
->Get("SIGNAL", 1)
.packets.push_back(Adopt(input_ocr.release()).At(Timestamp(0)));
MP_ASSERT_OK(runner->Run());
const std::vector<Packet>& output_packets =
runner->Outputs().Tag("OUTPUT").packets;
const auto& detection_set = output_packets[0].Get<DetectionSet>();
ASSERT_EQ(detection_set.detections().size(), 4);
EXPECT_FLOAT_EQ(detection_set.detections(0).score(), .55);
EXPECT_FLOAT_EQ(detection_set.detections(1).score(), .53);
EXPECT_FLOAT_EQ(detection_set.detections(2).score(), .93);
EXPECT_FLOAT_EQ(detection_set.detections(3).score(), .99);
}
TEST(SignalFusingCalculatorTest, TwoInputNoShotLabeledTags) {
auto runner = absl::make_unique<CalculatorRunner>(
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfigC));
auto input_face =
absl::make_unique<DetectionSet>(ParseTextProtoOrDie<DetectionSet>(
R"(
detections {
score: 0.5
signal_type: { standard: FACE_FULL }
}
detections {
score: 0.3
signal_type: { standard: FACE_FULL }
}
)"));
runner->MutableInputs()
->Get("SIGNAL", 0)
.packets.push_back(Adopt(input_face.release()).At(Timestamp(0)));
auto input_ocr =
absl::make_unique<DetectionSet>(ParseTextProtoOrDie<DetectionSet>(
R"(
detections {
score: 0.3
signal_type: { standard: TEXT }
}
detections {
score: 0.9
signal_type: { standard: TEXT }
}
)"));
runner->MutableInputs()
->Get("SIGNAL", 1)
.packets.push_back(Adopt(input_ocr.release()).At(Timestamp(0)));
MP_ASSERT_OK(runner->Run());
const std::vector<Packet>& output_packets =
runner->Outputs().Tag("OUTPUT").packets;
const auto& detection_set = output_packets[0].Get<DetectionSet>();
ASSERT_EQ(detection_set.detections().size(), 4);
EXPECT_FLOAT_EQ(detection_set.detections(0).score(), .55);
EXPECT_FLOAT_EQ(detection_set.detections(1).score(), .53);
EXPECT_FLOAT_EQ(detection_set.detections(2).score(), .93);
EXPECT_FLOAT_EQ(detection_set.detections(3).score(), .99);
}
TEST(SignalFusingCalculatorTest, ThreeInputTracking) {
auto runner = absl::make_unique<CalculatorRunner>(
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfigB));
mediapipe/examples/desktop/autoflip/quality/BUILD
+41 -1
View File
@@ -23,17 +23,33 @@ proto_library(
srcs = ["cropping.proto"],
deps = [
"//mediapipe/examples/desktop/autoflip:autoflip_messages_proto",
"//mediapipe/examples/desktop/autoflip/quality:kinematic_path_solver_proto",
],
)
mediapipe_cc_proto_library(
name = "cropping_cc_proto",
srcs = ["cropping.proto"],
cc_deps = ["//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto"],
cc_deps = [
":kinematic_path_solver_cc_proto",
"//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto",
],
visibility = ["//mediapipe/examples:__subpackages__"],
deps = [":cropping_proto"],
)
proto_library(
name = "kinematic_path_solver_proto",
srcs = ["kinematic_path_solver.proto"],
)
mediapipe_cc_proto_library(
name = "kinematic_path_solver_cc_proto",
srcs = ["kinematic_path_solver.proto"],
visibility = ["//mediapipe/examples:__subpackages__"],
deps = [":kinematic_path_solver_proto"],
)
proto_library(
name = "focus_point_proto",
srcs = ["focus_point.proto"],
@@ -142,6 +158,29 @@ cc_library(
],
)
cc_library(
name = "kinematic_path_solver",
srcs = ["kinematic_path_solver.cc"],
hdrs = ["kinematic_path_solver.h"],
deps = [
":kinematic_path_solver_cc_proto",
"//mediapipe/framework/port:integral_types",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
],
)
cc_test(
name = "kinematic_path_solver_test",
srcs = ["kinematic_path_solver_test.cc"],
deps = [
":kinematic_path_solver",
"//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:integral_types",
"//mediapipe/framework/port:status",
],
)
cc_library(
name = "scene_cropper",
srcs = ["scene_cropper.cc"],
@@ -149,6 +188,7 @@ cc_library(
deps = [
":cropping_cc_proto",
":focus_point_cc_proto",
":kinematic_path_solver",
":polynomial_regression_path_solver",
":utils",
"//mediapipe/framework/port:opencv_core",
mediapipe/examples/desktop/autoflip/quality/cropping.proto
+24
View File
@@ -17,6 +17,7 @@ syntax = "proto2";
package mediapipe.autoflip;
import "mediapipe/examples/desktop/autoflip/autoflip_messages.proto";
import "mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver.proto";
// All relevant information for key frames, including timestamp and detected
// features. This object should be generated by calling PackKeyFrameInfo() in
@@ -77,6 +78,8 @@ message KeyFrameCropResult {
optional Rect region = 6;
// Score of the computed crop region based on the detected features.
optional float region_score = 7;
// Frame timestamp (in microseconds).
optional int64 timestamp_ms = 8;
}
// Compact processed scene key frame info containing timestamp, center position,
@@ -172,6 +175,7 @@ message SceneCameraMotion {
// User-specified options for analyzing scene camera motion from a collection of
// key frame crop regions.
message SceneCameraMotionAnalyzerOptions {
reserved 9;
// If there is small motion within the scene keep the camera steady at the
// center.
optional float motion_stabilization_threshold_percent = 1 [default = .30];
@@ -196,6 +200,11 @@ message SceneCameraMotionAnalyzerOptions {
optional float minimum_success_rate_for_sweeping = 7 [default = 0.4];
// If true, sweep entire frame. Otherwise, sweep the crop window.
optional bool sweep_entire_frame = 8 [default = true];
// When no salient region is received, the default behavior is the return the
// camera to center-focused location. When this flag is set to a value >0,
// the camera will remain at its last position for this amount of time before
// recentering (if the last scene camera motion type was steady).
optional int64 duration_before_centering_us = 10;
}
// Video cropping summary information for debugging/statistics.
@@ -215,3 +224,18 @@ message VideoCroppingSummary {
// Cropping summaries for all the scenes in the video.
repeated SceneCroppingSummary scene_summaries = 1;
}
message CameraMotionOptions {
message PolynomialRegressionPathSolver {
// Number of frames from prior buffer to be used to smooth out camera
// trajectory when it was a forced flush.
optional int32 prior_frame_buffer_size = 1 [default = 30];
}
oneof camera_model_oneof {
// Fits a poly line to keypoints to find a smooth camera path.
PolynomialRegressionPathSolver polynomial_path_solver = 1;
// Maintains a kinematic state of the camera, updated with keypoints, to
// find a smooth camera path. Currently optimized for real-time operation.
KinematicOptions kinematic_options = 2;
}
}
mediapipe/examples/desktop/autoflip/quality/frame_crop_region_computer.cc
+3 -1
View File
@@ -171,6 +171,9 @@ void FrameCropRegionComputer::UpdateCropRegionScore(
const KeyFrameInfo& frame_info, KeyFrameCropResult* crop_result) const {
RET_CHECK(crop_result != nullptr) << "KeyFrameCropResult is null.";
// Set timestamp of KeyFrameCropResult
crop_result->set_timestamp_ms(frame_info.timestamp_ms());
// Sorts required and non-required regions.
std::vector<SalientRegion> required_regions, non_required_regions;
const auto sort_status = SortDetections(
@@ -180,7 +183,6 @@ void FrameCropRegionComputer::UpdateCropRegionScore(
int target_width = options_.target_width();
int target_height = options_.target_height();
auto* region = crop_result->mutable_region();
RET_CHECK(region != nullptr) << "Crop region is null.";
bool crop_region_is_empty = true;
float crop_region_score = 0.0;
mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver.cc
+76
View File
@@ -0,0 +1,76 @@
#include "mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver.h"
namespace mediapipe {
namespace autoflip {
::mediapipe::Status KinematicPathSolver::AddObservation(int position,
const uint64 time_us) {
if (!initialized_) {
current_position_px_ = position;
current_time_ = time_us;
initialized_ = true;
current_velocity_deg_per_s_ = 0;
return ::mediapipe::OkStatus();
}
RET_CHECK(current_time_ < time_us)
<< "Observation added before a prior observations.";
double delta_degs = (position - current_position_px_) / pixels_per_degree_;
// If the motion is smaller than the min, don't use the update.
if (abs(delta_degs) < options_.min_motion_to_reframe()) {
position = current_position_px_;
delta_degs = 0;
}
// Time and position updates.
double delta_t = (time_us - current_time_) / 1000000.0;
// Observed velocity and then weighted update of this velocity.
double observed_velocity = delta_degs / delta_t;
double updated_velocity =
current_velocity_deg_per_s_ * (1 - options_.update_rate()) +
observed_velocity * options_.update_rate();
// Limited current velocity.
current_velocity_deg_per_s_ =
updated_velocity > 0 ? fmin(updated_velocity, options_.max_velocity())
: fmax(updated_velocity, -options_.max_velocity());
// Update prediction based on time input.
return UpdatePrediction(time_us);
}
::mediapipe::Status KinematicPathSolver::UpdatePrediction(const int64 time_us) {
RET_CHECK(current_time_ < time_us)
<< "Prediction time added before a prior observation or prediction.";
// Time since last state/prediction update.
double delta_t = (time_us - current_time_) / 1000000.0;
// Position update limited by min/max.
const double update_position_px =
current_position_px_ +
current_velocity_deg_per_s_ * delta_t * pixels_per_degree_;
if (update_position_px < min_location_) {
current_position_px_ = min_location_;
current_velocity_deg_per_s_ = 0;
} else if (update_position_px > max_location_) {
current_position_px_ = max_location_;
current_velocity_deg_per_s_ = 0;
} else {
current_position_px_ = update_position_px;
}
current_time_ = time_us;
return ::mediapipe::OkStatus();
}
::mediapipe::Status KinematicPathSolver::GetState(int* position) {
RET_CHECK(initialized_) << "GetState called before first observation added.";
*position = round(current_position_px_);
return ::mediapipe::OkStatus();
}
} // namespace autoflip
} // namespace mediapipe
mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver.h
+67
View File
@@ -0,0 +1,67 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_UNIFORM_ACCELERATION_PATH_SOLVER_H_
#define MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_UNIFORM_ACCELERATION_PATH_SOLVER_H_
#include "mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver.pb.h"
#include "mediapipe/framework/port/integral_types.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status.h"
namespace mediapipe {
namespace autoflip {
// Kinematic path solver class is a stateful 1d position estimator based loosely
// on a differential kalman filter that is specifically designed to control a
// camera. It utilizes a Kalman filters predict/update interface for estimating
// the best camera focus position and updating that estimate when a measurement
// is available. Tuning controls include: update_rate: how much to update the
// existing state with a new state. max_velocity: max speed of the state per
// second. min_motion_to_reframe: only updating the state if a measurement
// exceeds this threshold.
class KinematicPathSolver {
public:
KinematicPathSolver(const KinematicOptions& options, const int min_location,
const int max_location, float pixels_per_degree)
: options_(options),
min_location_(min_location),
max_location_(max_location),
initialized_(false),
pixels_per_degree_(pixels_per_degree) {}
// Add an observation (detection) at a position and time.
::mediapipe::Status AddObservation(int position, const uint64 time_us);
// Get the predicted position at a time.
::mediapipe::Status UpdatePrediction(const int64 time_us);
// Get the state at a time.
::mediapipe::Status GetState(int* position);
private:
// Tuning options.
KinematicOptions options_;
// Min and max value the state can be.
const int min_location_;
const int max_location_;
bool initialized_;
float pixels_per_degree_;
// Current state values.
double current_position_px_;
double current_velocity_deg_per_s_;
uint64 current_time_;
};
} // namespace autoflip
} // namespace mediapipe
#endif // MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_UNIFORM_ACCELERATION_PATH_SOLVER_H_
mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver.proto
+13
View File
@@ -0,0 +1,13 @@
syntax = "proto2";
package mediapipe.autoflip;
message KinematicOptions {
// Weighted update of new camera velocity (measurement) vs current state
// (prediction).
optional double update_rate = 1 [default = 0.5];
// Max velocity (degrees per second) that the camera can move.
optional double max_velocity = 2 [default = 18];
// Min motion (in degrees) to react in pixels.
optional float min_motion_to_reframe = 3 [default = 1.8];
}
mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver_test.cc
+140
View File
@@ -0,0 +1,140 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver.h"
#include "mediapipe/framework/port/gmock.h"
#include "mediapipe/framework/port/gtest.h"
#include "mediapipe/framework/port/integral_types.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/framework/port/status_matchers.h"
constexpr int64 kMicroSecInSec = 1000000;
constexpr float kWidthFieldOfView = 60;
namespace mediapipe {
namespace autoflip {
namespace {
TEST(KinematicPathSolverTest, FailNotInitializedState) {
KinematicOptions options;
KinematicPathSolver solver(options, 0, 1000, 1000.0 / kWidthFieldOfView);
int state;
EXPECT_FALSE(solver.GetState(&state).ok());
}
TEST(KinematicPathSolverTest, FailNotInitializedPrediction) {
KinematicOptions options;
KinematicPathSolver solver(options, 0, 1000, 1000.0 / kWidthFieldOfView);
int64 timestamp = 0;
EXPECT_FALSE(solver.UpdatePrediction(timestamp).ok());
}
TEST(KinematicPathSolverTest, PassNotEnoughMotionLargeImg) {
KinematicOptions options;
// Set min motion to 2deg
options.set_min_motion_to_reframe(2.0);
options.set_update_rate(1);
options.set_max_velocity(1000);
// Set degrees / pixel to 16.6
KinematicPathSolver solver(options, 0, 1000, 1000.0 / kWidthFieldOfView);
int state;
MP_ASSERT_OK(solver.AddObservation(500, kMicroSecInSec * 0));
// Move target by 20px / 16.6 = 1.2deg
MP_ASSERT_OK(solver.AddObservation(520, kMicroSecInSec * 1));
MP_ASSERT_OK(solver.GetState(&state));
// Expect cam to not move.
EXPECT_EQ(state, 500);
}
TEST(KinematicPathSolverTest, PassNotEnoughMotionSmallImg) {
KinematicOptions options;
// Set min motion to 2deg
options.set_min_motion_to_reframe(2.0);
options.set_update_rate(1);
options.set_max_velocity(500);
// Set degrees / pixel to 8.3
KinematicPathSolver solver(options, 0, 500, 500.0 / kWidthFieldOfView);
int state;
MP_ASSERT_OK(solver.AddObservation(400, kMicroSecInSec * 0));
// Move target by 10px / 8.3 = 1.2deg
MP_ASSERT_OK(solver.AddObservation(410, kMicroSecInSec * 1));
MP_ASSERT_OK(solver.GetState(&state));
// Expect cam to not move.
EXPECT_EQ(state, 400);
}
TEST(KinematicPathSolverTest, PassEnoughMotionLargeImg) {
KinematicOptions options;
// Set min motion to 1deg
options.set_min_motion_to_reframe(1.0);
options.set_update_rate(1);
options.set_max_velocity(1000);
// Set degrees / pixel to 16.6
KinematicPathSolver solver(options, 0, 1000, 1000.0 / kWidthFieldOfView);
int state;
MP_ASSERT_OK(solver.AddObservation(500, kMicroSecInSec * 0));
// Move target by 20px / 16.6 = 1.2deg
MP_ASSERT_OK(solver.AddObservation(520, kMicroSecInSec * 1));
MP_ASSERT_OK(solver.GetState(&state));
// Expect cam to move.
EXPECT_EQ(state, 520);
}
TEST(KinematicPathSolverTest, PassEnoughMotionSmallImg) {
KinematicOptions options;
// Set min motion to 2deg
options.set_min_motion_to_reframe(1.0);
options.set_update_rate(1);
options.set_max_velocity(18);
// Set degrees / pixel to 8.3
KinematicPathSolver solver(options, 0, 500, 500.0 / kWidthFieldOfView);
int state;
MP_ASSERT_OK(solver.AddObservation(400, kMicroSecInSec * 0));
// Move target by 10px / 8.3 = 1.2deg
MP_ASSERT_OK(solver.AddObservation(410, kMicroSecInSec * 1));
MP_ASSERT_OK(solver.GetState(&state));
// Expect cam to move.
EXPECT_EQ(state, 410);
}
TEST(KinematicPathSolverTest, PassUpdateRate) {
KinematicOptions options;
options.set_min_motion_to_reframe(1.0);
options.set_update_rate(0.25);
options.set_max_velocity(18);
KinematicPathSolver solver(options, 0, 1000, 1000.0 / kWidthFieldOfView);
int state;
MP_ASSERT_OK(solver.AddObservation(500, kMicroSecInSec * 0));
MP_ASSERT_OK(solver.AddObservation(520, kMicroSecInSec * 1));
MP_ASSERT_OK(solver.GetState(&state));
EXPECT_EQ(state, 505);
}
TEST(KinematicPathSolverTest, PassMaxVelocity) {
KinematicOptions options;
options.set_min_motion_to_reframe(1.0);
options.set_update_rate(1.0);
options.set_max_velocity(6);
KinematicPathSolver solver(options, 0, 1000, 1000.0 / kWidthFieldOfView);
int state;
MP_ASSERT_OK(solver.AddObservation(500, kMicroSecInSec * 0));
MP_ASSERT_OK(solver.AddObservation(1000, kMicroSecInSec * 1));
MP_ASSERT_OK(solver.GetState(&state));
EXPECT_EQ(state, 600);
}
} // namespace
} // namespace autoflip
} // namespace mediapipe
mediapipe/examples/desktop/autoflip/quality/scene_camera_motion_analyzer.cc
+31 -12
View File
@@ -32,17 +32,19 @@ namespace autoflip {
::mediapipe::Status
SceneCameraMotionAnalyzer::AnalyzeSceneAndPopulateFocusPointFrames(
const std::vector<KeyFrameInfo>& key_frame_infos,
const KeyFrameCropOptions& key_frame_crop_options,
const std::vector<KeyFrameCropResult>& key_frame_crop_results,
const int scene_frame_width, const int scene_frame_height,
const std::vector<int64>& scene_frame_timestamps,
const bool has_solid_color_background,
SceneKeyFrameCropSummary* scene_summary,
std::vector<FocusPointFrame>* focus_point_frames,
SceneCameraMotion* scene_camera_motion) const {
SceneCameraMotion* scene_camera_motion) {
has_solid_color_background_ = has_solid_color_background;
total_scene_frames_ = scene_frame_timestamps.size();
MP_RETURN_IF_ERROR(AggregateKeyFrameResults(
key_frame_infos, key_frame_crop_options, key_frame_crop_results,
scene_frame_width, scene_frame_height, scene_summary));
key_frame_crop_options, key_frame_crop_results, scene_frame_width,
scene_frame_height, scene_summary));
const int64 scene_span_ms =
scene_frame_timestamps.empty()
@@ -51,7 +53,12 @@ SceneCameraMotionAnalyzer::AnalyzeSceneAndPopulateFocusPointFrames(
const double scene_span_sec = TimestampDiff(scene_span_ms).Seconds();
SceneCameraMotion camera_motion;
MP_RETURN_IF_ERROR(DecideCameraMotionType(
key_frame_crop_options, scene_span_sec, scene_summary, &camera_motion));
key_frame_crop_options, scene_span_sec, scene_frame_timestamps.back(),
scene_summary, &camera_motion));
if (scene_summary->has_salient_region()) {
last_scene_with_salient_region_ = camera_motion;
time_since_last_salient_region_us_ = scene_frame_timestamps.back();
}
if (scene_camera_motion != nullptr) {
*scene_camera_motion = camera_motion;
}
@@ -97,7 +104,8 @@ SceneCameraMotionAnalyzer::AnalyzeSceneAndPopulateFocusPointFrames(
::mediapipe::Status SceneCameraMotionAnalyzer::DecideCameraMotionType(
const KeyFrameCropOptions& key_frame_crop_options,
const double scene_span_sec, SceneKeyFrameCropSummary* scene_summary,
const double scene_span_sec, const int64 end_time_us,
SceneKeyFrameCropSummary* scene_summary,
SceneCameraMotion* scene_camera_motion) const {
RET_CHECK_GE(scene_span_sec, 0.0) << "Scene time span is negative.";
RET_CHECK_NE(scene_summary, nullptr) << "Scene summary is null.";
@@ -109,8 +117,18 @@ SceneCameraMotionAnalyzer::AnalyzeSceneAndPopulateFocusPointFrames(
// regions, then default to look at the center.
if (!scene_summary->has_salient_region()) {
VLOG(1) << "No focus regions - camera is set to be steady on center.";
float no_salient_position_x = scene_frame_center_x;
float no_salient_position_y = scene_frame_center_y;
if (end_time_us - time_since_last_salient_region_us_ <
options_.duration_before_centering_us() &&
last_scene_with_salient_region_.has_steady_motion()) {
no_salient_position_x = last_scene_with_salient_region_.steady_motion()
.steady_look_at_center_x();
no_salient_position_y = last_scene_with_salient_region_.steady_motion()
.steady_look_at_center_y();
}
MP_RETURN_IF_ERROR(ToUseSteadyMotion(
scene_frame_center_x, scene_frame_center_y,
no_salient_position_x, no_salient_position_y,
scene_summary->crop_window_width(), scene_summary->crop_window_height(),
scene_summary, scene_camera_motion));
return ::mediapipe::OkStatus();
@@ -118,7 +136,7 @@ SceneCameraMotionAnalyzer::AnalyzeSceneAndPopulateFocusPointFrames(
// Sweep across the scene when 1) success rate is too low, AND 2) the current
// scene is long enough.
if (options_.allow_sweeping() &&
if (options_.allow_sweeping() && !has_solid_color_background_ &&
scene_summary->frame_success_rate() <
options_.minimum_success_rate_for_sweeping() &&
scene_span_sec >= options_.minimum_scene_span_sec_for_sweeping()) {
@@ -150,10 +168,11 @@ SceneCameraMotionAnalyzer::AnalyzeSceneAndPopulateFocusPointFrames(
}
// If scene motion is small, then look at a steady point in the scene.
if (scene_summary->horizontal_motion_amount() <
options_.motion_stabilization_threshold_percent() &&
scene_summary->vertical_motion_amount() <
options_.motion_stabilization_threshold_percent()) {
if ((scene_summary->horizontal_motion_amount() <
options_.motion_stabilization_threshold_percent() &&
scene_summary->vertical_motion_amount() <
options_.motion_stabilization_threshold_percent()) ||
total_scene_frames_ == 1) {
return DecideSteadyLookAtRegion(key_frame_crop_options, scene_summary,
scene_camera_motion);
}
mediapipe/examples/desktop/autoflip/quality/scene_camera_motion_analyzer.h
+17 -4
View File
@@ -53,7 +53,9 @@ class SceneCameraMotionAnalyzer {
explicit SceneCameraMotionAnalyzer(const SceneCameraMotionAnalyzerOptions&
scene_camera_motion_analyzer_options)
: options_(scene_camera_motion_analyzer_options) {}
: options_(scene_camera_motion_analyzer_options),
time_since_last_salient_region_us_(0),
has_solid_color_background_(false) {}
~SceneCameraMotionAnalyzer() {}
@@ -61,21 +63,22 @@ class SceneCameraMotionAnalyzer {
// SceneKeyFrameCropSummary, and populates FocusPointFrames given scene
// frame timestamps. Optionally returns SceneCameraMotion.
::mediapipe::Status AnalyzeSceneAndPopulateFocusPointFrames(
const std::vector<KeyFrameInfo>& key_frame_infos,
const KeyFrameCropOptions& key_frame_crop_options,
const std::vector<KeyFrameCropResult>& key_frame_crop_results,
const int scene_frame_width, const int scene_frame_height,
const std::vector<int64>& scene_frame_timestamps,
const bool has_solid_color_background,
SceneKeyFrameCropSummary* scene_summary,
std::vector<FocusPointFrame>* focus_point_frames,
SceneCameraMotion* scene_camera_motion = nullptr) const;
SceneCameraMotion* scene_camera_motion = nullptr);
protected:
// Decides SceneCameraMotion based on SceneKeyFrameCropSummary. Updates the
// crop window in SceneKeyFrameCropSummary in the case of steady motion.
::mediapipe::Status DecideCameraMotionType(
const KeyFrameCropOptions& key_frame_crop_options,
const double scene_span_sec, SceneKeyFrameCropSummary* scene_summary,
const double scene_span_sec, const int64 end_time_us,
SceneKeyFrameCropSummary* scene_summary,
SceneCameraMotion* scene_camera_motion) const;
// Populates the FocusPointFrames for each scene frame based on
@@ -134,6 +137,16 @@ class SceneCameraMotionAnalyzer {
// Scene camera motion analyzer options.
SceneCameraMotionAnalyzerOptions options_;
// Last position
SceneCameraMotion last_scene_with_salient_region_;
int64 time_since_last_salient_region_us_;
// Scene has solid color background.
bool has_solid_color_background_;
// Total number of frames for this scene.
int total_scene_frames_;
};
} // namespace autoflip
mediapipe/examples/desktop/autoflip/quality/scene_camera_motion_analyzer_test.cc
+20 -30
View File
@@ -63,16 +63,6 @@ Rect MakeRect(const int x, const int y, const int width, const int height) {
return rect;
}
// Returns default values for KeyFrameInfos. Populates timestamps using the
// default spacing kKeyFrameTimestampDiff starting from 0.
std::vector<KeyFrameInfo> GetDefaultKeyFrameInfos() {
std::vector<KeyFrameInfo> key_frame_infos(kNumKeyFrames);
for (int i = 0; i < kNumKeyFrames; ++i) {
key_frame_infos[i].set_timestamp_ms(kKeyFrameTimestampDiff * i);
}
return key_frame_infos;
}
// Returns default values for scene frame timestamps. Populates timestamps using
// the default spacing kSceneFrameTimestampDiff starting from 0.
std::vector<int64> GetDefaultSceneFrameTimestamps() {
@@ -108,6 +98,7 @@ std::vector<KeyFrameCropResult> GetDefaultKeyFrameCropResults() {
*(key_frame_crop_results[i].mutable_required_region()) =
MakeRect(10, 10, 20, 20);
key_frame_crop_results[i].set_region_score(1.0);
key_frame_crop_results[i].set_timestamp_ms(kKeyFrameTimestampDiff * i);
}
return key_frame_crop_results;
}
@@ -183,10 +174,10 @@ TEST(SceneCameraMotionAnalyzerTest, DecideCameraMotionTypeChecksOutputNotNull) {
SceneKeyFrameCropSummary scene_summary;
SceneCameraMotion camera_motion;
auto status = analyzer.DecideCameraMotionType(crop_options, kSceneTimeSpanSec,
nullptr, &camera_motion);
0, nullptr, &camera_motion);
EXPECT_FALSE(status.ok());
EXPECT_THAT(status.ToString(), HasSubstr("Scene summary is null."));
status = analyzer.DecideCameraMotionType(crop_options, kSceneTimeSpanSec,
status = analyzer.DecideCameraMotionType(crop_options, kSceneTimeSpanSec, 0,
&scene_summary, nullptr);
EXPECT_FALSE(status.ok());
EXPECT_THAT(status.ToString(), HasSubstr("Scene camera motion is null."));
@@ -204,8 +195,8 @@ TEST(SceneCameraMotionAnalyzerTest,
scene_summary.set_has_salient_region(false);
SceneCameraMotion camera_motion;
MP_EXPECT_OK(analyzer.DecideCameraMotionType(crop_options, kSceneTimeSpanSec,
&scene_summary, &camera_motion));
MP_EXPECT_OK(analyzer.DecideCameraMotionType(
crop_options, kSceneTimeSpanSec, 0, &scene_summary, &camera_motion));
EXPECT_TRUE(camera_motion.has_steady_motion());
const auto& steady_motion = camera_motion.steady_motion();
EXPECT_FLOAT_EQ(steady_motion.steady_look_at_center_x(),
@@ -229,7 +220,7 @@ TEST(SceneCameraMotionAnalyzerTest, DecideCameraMotionTypeSweepingLeftToRight) {
SceneCameraMotion camera_motion;
MP_EXPECT_OK(analyzer.DecideCameraMotionType(GetDefaultKeyFrameCropOptions(),
time_span, &scene_summary,
time_span, 0, &scene_summary,
&camera_motion));
EXPECT_TRUE(camera_motion.has_sweeping_motion());
@@ -258,7 +249,7 @@ TEST(SceneCameraMotionAnalyzerTest, DecideCameraMotionTypeSweepingTopToBottom) {
SceneCameraMotion camera_motion;
MP_EXPECT_OK(analyzer.DecideCameraMotionType(GetDefaultKeyFrameCropOptions(),
time_span, &scene_summary,
time_span, 0, &scene_summary,
&camera_motion));
EXPECT_TRUE(camera_motion.has_sweeping_motion());
@@ -287,7 +278,7 @@ TEST(SceneCameraMotionAnalyzerTest, DecideCameraMotionTypeSweepingCenterRange) {
SceneCameraMotion camera_motion;
MP_EXPECT_OK(analyzer.DecideCameraMotionType(GetDefaultKeyFrameCropOptions(),
time_span, &scene_summary,
time_span, 0, &scene_summary,
&camera_motion));
EXPECT_TRUE(camera_motion.has_sweeping_motion());
@@ -316,7 +307,7 @@ TEST(SceneCameraMotionAnalyzerTest,
SceneCameraMotion camera_motion;
MP_EXPECT_OK(analyzer.DecideCameraMotionType(GetDefaultKeyFrameCropOptions(),
kSceneTimeSpanSec,
kSceneTimeSpanSec, 0,
&scene_summary, &camera_motion));
EXPECT_TRUE(camera_motion.has_steady_motion());
EXPECT_EQ(camera_motion.steady_motion().steady_look_at_center_x(),
@@ -341,7 +332,7 @@ TEST(SceneCameraMotionAnalyzerTest,
SceneCameraMotion camera_motion;
MP_EXPECT_OK(analyzer.DecideCameraMotionType(GetDefaultKeyFrameCropOptions(),
kSceneTimeSpanSec,
kSceneTimeSpanSec, 0,
&scene_summary, &camera_motion));
EXPECT_TRUE(camera_motion.has_steady_motion());
EXPECT_EQ(camera_motion.steady_motion().steady_look_at_center_x(),
@@ -366,7 +357,7 @@ TEST(SceneCameraMotionAnalyzerTest,
SceneCameraMotion camera_motion;
MP_EXPECT_OK(analyzer.DecideCameraMotionType(GetDefaultKeyFrameCropOptions(),
kSceneTimeSpanSec,
kSceneTimeSpanSec, 0,
&scene_summary, &camera_motion));
EXPECT_TRUE(camera_motion.has_steady_motion());
EXPECT_EQ(camera_motion.steady_motion().steady_look_at_center_x(),
@@ -391,8 +382,8 @@ TEST(SceneCameraMotionAnalyzerTest,
scene_summary.set_key_frame_center_max_x(frame_center_x);
SceneCameraMotion camera_motion;
MP_EXPECT_OK(analyzer.DecideCameraMotionType(crop_options, kSceneTimeSpanSec,
&scene_summary, &camera_motion));
MP_EXPECT_OK(analyzer.DecideCameraMotionType(
crop_options, kSceneTimeSpanSec, 0, &scene_summary, &camera_motion));
EXPECT_TRUE(camera_motion.has_steady_motion());
EXPECT_FLOAT_EQ(camera_motion.steady_motion().steady_look_at_center_x(),
frame_center_x);
@@ -409,7 +400,7 @@ TEST(SceneCameraMotionAnalyzerTest, DecideCameraMotionTypeTracking) {
SceneCameraMotion camera_motion;
MP_EXPECT_OK(analyzer.DecideCameraMotionType(GetDefaultKeyFrameCropOptions(),
kSceneTimeSpanSec,
kSceneTimeSpanSec, 0,
&scene_summary, &camera_motion));
EXPECT_TRUE(camera_motion.has_tracking_motion());
}
@@ -785,9 +776,9 @@ TEST(SceneCameraMotionAnalyzerTest, AnalyzeSceneAndPopulateFocusPointFrames) {
std::vector<FocusPointFrame> focus_point_frames;
MP_EXPECT_OK(analyzer.AnalyzeSceneAndPopulateFocusPointFrames(
GetDefaultKeyFrameInfos(), GetDefaultKeyFrameCropOptions(),
GetDefaultKeyFrameCropResults(), kSceneFrameWidth, kSceneFrameHeight,
GetDefaultSceneFrameTimestamps(), &scene_summary, &focus_point_frames));
GetDefaultKeyFrameCropOptions(), GetDefaultKeyFrameCropResults(),
kSceneFrameWidth, kSceneFrameHeight, GetDefaultSceneFrameTimestamps(),
false, &scene_summary, &focus_point_frames));
EXPECT_EQ(scene_summary.num_key_frames(), kNumKeyFrames);
EXPECT_EQ(focus_point_frames.size(), kNumSceneFrames);
}
@@ -803,10 +794,9 @@ TEST(SceneCameraMotionAnalyzerTest,
SceneCameraMotion scene_camera_motion;
MP_EXPECT_OK(analyzer.AnalyzeSceneAndPopulateFocusPointFrames(
GetDefaultKeyFrameInfos(), GetDefaultKeyFrameCropOptions(),
GetDefaultKeyFrameCropResults(), kSceneFrameWidth, kSceneFrameHeight,
GetDefaultSceneFrameTimestamps(), &scene_summary, &focus_point_frames,
&scene_camera_motion));
GetDefaultKeyFrameCropOptions(), GetDefaultKeyFrameCropResults(),
kSceneFrameWidth, kSceneFrameHeight, GetDefaultSceneFrameTimestamps(),
false, &scene_summary, &focus_point_frames, &scene_camera_motion));
EXPECT_TRUE(scene_camera_motion.has_steady_motion());
}
mediapipe/examples/desktop/autoflip/quality/scene_cropper.cc
+102 -33
View File
@@ -14,6 +14,8 @@
#include "mediapipe/examples/desktop/autoflip/quality/scene_cropper.h"
#include <memory>
#include "absl/memory/memory.h"
#include "mediapipe/examples/desktop/autoflip/quality/polynomial_regression_path_solver.h"
#include "mediapipe/examples/desktop/autoflip/quality/utils.h"
@@ -21,17 +23,74 @@
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status.h"
// TODO: Parameterize FOV based on camera specs.
constexpr float kWidthFieldOfView = 60;
namespace mediapipe {
namespace autoflip {
::mediapipe::Status SceneCropper::ProcessKinematicPathSolver(
const SceneKeyFrameCropSummary& scene_summary,
const std::vector<int64>& scene_timestamps,
const std::vector<bool>& is_key_frames,
const std::vector<FocusPointFrame>& focus_point_frames,
const bool continue_last_scene, std::vector<cv::Mat>* all_xforms) {
// TODO: Fix upstream calculators to not crop beyond portrait target
// value.
/*
RET_CHECK(scene_summary.scene_frame_height() ==
scene_summary.crop_window_height())
<< "Kinematic path solver does not yet support horizontal cropping.";
*/
RET_CHECK(scene_timestamps.size() == focus_point_frames.size())
<< "Kinematic path solver does not yet support downsampled detections.";
if (!path_solver_initalized_ || !continue_last_scene) {
int min_location = scene_summary.crop_window_width() / 2;
int max_location = scene_summary.scene_frame_width() -
scene_summary.crop_window_width() / 2;
kinematic_path_solver_ = std::make_unique<KinematicPathSolver>(
camera_motion_options_.kinematic_options(), min_location, max_location,
static_cast<float>(frame_width_) / kWidthFieldOfView);
path_solver_initalized_ = true;
}
int keyframe_counter = 0;
for (int i = 0; i < is_key_frames.size(); i++) {
if (is_key_frames[i]) {
RET_CHECK_EQ(focus_point_frames[keyframe_counter].point().size(), 2)
<< "Expected focus_points to equal 2";
int observed_x = std::round(
focus_point_frames[keyframe_counter].point(0).norm_point_x() *
scene_summary.scene_frame_width());
MP_RETURN_IF_ERROR(kinematic_path_solver_->AddObservation(
observed_x, scene_timestamps[i]));
keyframe_counter++;
} else {
MP_RETURN_IF_ERROR(
kinematic_path_solver_->UpdatePrediction(scene_timestamps[i]));
}
int x_path;
MP_RETURN_IF_ERROR(kinematic_path_solver_->GetState(&x_path));
cv::Mat transform = cv::Mat::eye(2, 3, CV_32FC1);
transform.at<float>(0, 2) =
-(x_path - scene_summary.crop_window_width() / 2);
all_xforms->push_back(transform);
}
return ::mediapipe::OkStatus();
}
::mediapipe::Status SceneCropper::CropFrames(
const SceneKeyFrameCropSummary& scene_summary, const int num_scene_frames,
const SceneKeyFrameCropSummary& scene_summary,
const std::vector<int64>& scene_timestamps,
const std::vector<bool>& is_key_frames,
const std::vector<cv::Mat>& scene_frames_or_empty,
const std::vector<FocusPointFrame>& focus_point_frames,
const std::vector<FocusPointFrame>& prior_focus_point_frames,
int top_static_border_size, int bottom_static_border_size,
std::vector<cv::Rect>* crop_from_location,
std::vector<cv::Mat>* cropped_frames) const {
const bool continue_last_scene, std::vector<cv::Rect>* crop_from_location,
std::vector<cv::Mat>* cropped_frames) {
const int num_scene_frames = scene_timestamps.size();
RET_CHECK_GT(num_scene_frames, 0) << "No scene frames.";
RET_CHECK_EQ(focus_point_frames.size(), num_scene_frames)
<< "Wrong size of FocusPointFrames.";
@@ -46,26 +105,50 @@ namespace autoflip {
RET_CHECK_LE(crop_height, frame_height)
<< "Crop height exceeds frame height.";
RET_CHECK(camera_motion_options_.has_polynomial_path_solver() ||
camera_motion_options_.has_kinematic_options())
<< "No camera motion model selected.";
// Computes transforms.
std::vector<cv::Mat> all_xforms;
PolynomialRegressionPathSolver solver;
RET_CHECK_OK(solver.ComputeCameraPath(
focus_point_frames, prior_focus_point_frames, frame_width, frame_height,
crop_width, crop_height, &all_xforms));
std::vector<cv::Mat> scene_frame_xforms;
int num_prior = 0;
if (camera_motion_options_.has_polynomial_path_solver()) {
num_prior = prior_focus_point_frames.size();
std::vector<cv::Mat> all_xforms;
PolynomialRegressionPathSolver solver;
RET_CHECK_OK(solver.ComputeCameraPath(
focus_point_frames, prior_focus_point_frames, frame_width, frame_height,
crop_width, crop_height, &all_xforms));
const int num_prior = prior_focus_point_frames.size();
std::vector<cv::Mat> scene_frame_xforms(all_xforms.begin() + num_prior,
all_xforms.end());
scene_frame_xforms =
std::vector<cv::Mat>(all_xforms.begin() + num_prior, all_xforms.end());
// Convert the matrix from center-aligned to upper-left aligned.
for (cv::Mat& xform : scene_frame_xforms) {
cv::Mat affine_opencv = cv::Mat::eye(2, 3, CV_32FC1);
affine_opencv.at<float>(0, 2) =
-(xform.at<float>(0, 2) + frame_width / 2 - crop_width / 2);
affine_opencv.at<float>(1, 2) =
-(xform.at<float>(1, 2) + frame_height / 2 - crop_height / 2);
xform = affine_opencv;
// Convert the matrix from center-aligned to upper-left aligned.
for (cv::Mat& xform : scene_frame_xforms) {
cv::Mat affine_opencv = cv::Mat::eye(2, 3, CV_32FC1);
affine_opencv.at<float>(0, 2) =
-(xform.at<float>(0, 2) + frame_width / 2 - crop_width / 2);
affine_opencv.at<float>(1, 2) =
-(xform.at<float>(1, 2) + frame_height / 2 - crop_height / 2);
xform = affine_opencv;
}
} else if (camera_motion_options_.has_kinematic_options()) {
num_prior = 0;
MP_RETURN_IF_ERROR(ProcessKinematicPathSolver(
scene_summary, scene_timestamps, is_key_frames, focus_point_frames,
continue_last_scene, &scene_frame_xforms));
}
// Store the "crop from" location on the input frame for use with an external
// renderer.
for (int i = 0; i < num_scene_frames; i++) {
const int left = -(scene_frame_xforms[i].at<float>(0, 2));
const int right = left + crop_width;
const int top = top_static_border_size;
const int bottom = frame_height_ - bottom_static_border_size;
crop_from_location->push_back(
cv::Rect(left, top, right - left, bottom - top));
}
// If no cropped_frames is passed in, return directly.
@@ -81,20 +164,6 @@ namespace autoflip {
(*cropped_frames)[i] = cv::Mat::zeros(crop_height, crop_width,
scene_frames_or_empty[i].type());
}
// Store the "crop from" location on the input frame for use with an external
// renderer.
for (int i = 0; i < num_scene_frames; i++) {
const int left = scene_frame_xforms[i].at<float>(0, 2);
const int right = left + crop_width;
const int top = top_static_border_size;
const int bottom =
top_static_border_size +
(crop_height - top_static_border_size - bottom_static_border_size);
crop_from_location->push_back(
cv::Rect(left, top, right - left, bottom - top));
}
return AffineRetarget(cv::Size(crop_width, crop_height),
scene_frames_or_empty, scene_frame_xforms,
cropped_frames);
mediapipe/examples/desktop/autoflip/quality/scene_cropper.h
+26 -4
View File
@@ -20,6 +20,7 @@
#include "mediapipe/examples/desktop/autoflip/quality/cropping.pb.h"
#include "mediapipe/examples/desktop/autoflip/quality/focus_point.pb.h"
#include "mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver.h"
#include "mediapipe/framework/port/opencv_core_inc.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status.h"
@@ -45,7 +46,12 @@ namespace autoflip {
// prior_focus_point_frames, &cropped_frames));
class SceneCropper {
public:
SceneCropper() {}
SceneCropper(const CameraMotionOptions& camera_motion_options,
const int frame_width, const int frame_height)
: path_solver_initalized_(false),
camera_motion_options_(camera_motion_options),
frame_width_(frame_width),
frame_height_(frame_height) {}
~SceneCropper() {}
// Computes transformation matrix given SceneKeyFrameCropSummary,
@@ -55,13 +61,29 @@ class SceneCropper {
// |scene_frames_or_empty| isn't empty.
// TODO: split this function into two separate functions.
::mediapipe::Status CropFrames(
const SceneKeyFrameCropSummary& scene_summary, const int num_scene_frames,
const SceneKeyFrameCropSummary& scene_summary,
const std::vector<int64>& scene_timestamps,
const std::vector<bool>& is_key_frames,
const std::vector<cv::Mat>& scene_frames_or_empty,
const std::vector<FocusPointFrame>& focus_point_frames,
const std::vector<FocusPointFrame>& prior_focus_point_frames,
int top_static_border_size, int bottom_static_border_size,
std::vector<cv::Rect>* all_scene_frame_xforms,
std::vector<cv::Mat>* cropped_frames) const;
const bool continue_last_scene, std::vector<cv::Rect>* crop_from_location,
std::vector<cv::Mat>* cropped_frames);
::mediapipe::Status ProcessKinematicPathSolver(
const SceneKeyFrameCropSummary& scene_summary,
const std::vector<int64>& scene_timestamps,
const std::vector<bool>& is_key_frames,
const std::vector<FocusPointFrame>& focus_point_frames,
const bool continue_last_scene, std::vector<cv::Mat>* all_xforms);
private:
bool path_solver_initalized_;
std::unique_ptr<KinematicPathSolver> kinematic_path_solver_;
CameraMotionOptions camera_motion_options_;
int frame_width_;
int frame_height_;
};
} // namespace autoflip
mediapipe/examples/desktop/autoflip/quality/scene_cropper_test.cc
+53 -18
View File
@@ -71,30 +71,53 @@ std::vector<FocusPointFrame> GetDefaultFocusPointFrames() {
return GetFocusPointFrames(kNumSceneFrames);
}
std::vector<int64> GetTimestamps(const int num_frames) {
std::vector<int64> timestamps;
for (int i = 0; i < num_frames; ++i) {
timestamps.push_back(i * 100000);
}
return timestamps;
}
std::vector<bool> GetIsKeyframe(const int num_frames) {
std::vector<bool> is_keyframe;
for (int i = 0; i < num_frames; ++i) {
is_keyframe.push_back(false);
}
return is_keyframe;
}
// Checks that CropFrames checks that scene frames size is positive.
TEST(SceneCropperTest, CropFramesChecksSceneFramesSize) {
SceneCropper scene_cropper;
CameraMotionOptions options;
options.mutable_polynomial_path_solver()->set_prior_frame_buffer_size(30);
SceneCropper scene_cropper(options, kSceneWidth, kSceneHeight);
std::vector<cv::Mat> scene_frames(0);
std::vector<cv::Mat> cropped_frames;
std::vector<cv::Rect> crop_from_locations;
const auto status = scene_cropper.CropFrames(
GetDefaultSceneKeyFrameCropSummary(), scene_frames.size(), scene_frames,
GetDefaultFocusPointFrames(), GetFocusPointFrames(0), 0, 0,
GetDefaultSceneKeyFrameCropSummary(), GetTimestamps(scene_frames.size()),
GetIsKeyframe(scene_frames.size()), scene_frames,
GetDefaultFocusPointFrames(), GetFocusPointFrames(0), 0, 0, false,
&crop_from_locations, &cropped_frames);
EXPECT_FALSE(status.ok());
EXPECT_THAT(status.ToString(), HasSubstr("No scene frames."));
}
// Checks that CropFrames checks that FocusPointFrames has the right size.
TEST(SceneCropperTest, CropFramesChecksFocusPointFramesSize) {
SceneCropper scene_cropper;
CameraMotionOptions options;
options.mutable_polynomial_path_solver()->set_prior_frame_buffer_size(30);
SceneCropper scene_cropper(options, kSceneWidth, kSceneHeight);
std::vector<cv::Mat> cropped_frames;
std::vector<cv::Rect> crop_from_locations;
const auto& scene_frames = GetDefaultSceneFrames();
const auto status = scene_cropper.CropFrames(
GetDefaultSceneKeyFrameCropSummary(), scene_frames.size(), scene_frames,
GetDefaultSceneKeyFrameCropSummary(), GetTimestamps(kNumSceneFrames),
GetIsKeyframe(kNumSceneFrames), scene_frames,
GetFocusPointFrames(kNumSceneFrames - 1), GetFocusPointFrames(0), 0, 0,
&crop_from_locations, &cropped_frames);
false, &crop_from_locations, &cropped_frames);
EXPECT_FALSE(status.ok());
EXPECT_THAT(status.ToString(), HasSubstr("Wrong size of FocusPointFrames"));
}
@@ -103,13 +126,16 @@ TEST(SceneCropperTest, CropFramesChecksFocusPointFramesSize) {
TEST(SceneCropperTest, CropFramesChecksCropSizePositive) {
auto scene_summary = GetDefaultSceneKeyFrameCropSummary();
scene_summary.set_crop_window_width(-1);
SceneCropper scene_cropper;
CameraMotionOptions options;
options.mutable_polynomial_path_solver()->set_prior_frame_buffer_size(30);
SceneCropper scene_cropper(options, kSceneWidth, kSceneHeight);
std::vector<cv::Mat> cropped_frames;
std::vector<cv::Rect> crop_from_locations;
const auto& scene_frames = GetDefaultSceneFrames();
const auto status = scene_cropper.CropFrames(
scene_summary, scene_frames.size(), scene_frames,
GetDefaultFocusPointFrames(), GetFocusPointFrames(0), 0, 0,
scene_summary, GetTimestamps(kNumSceneFrames),
GetIsKeyframe(kNumSceneFrames), scene_frames,
GetDefaultFocusPointFrames(), GetFocusPointFrames(0), 0, 0, false,
&crop_from_locations, &cropped_frames);
EXPECT_FALSE(status.ok());
EXPECT_THAT(status.ToString(), HasSubstr("Crop width is non-positive."));
@@ -119,13 +145,16 @@ TEST(SceneCropperTest, CropFramesChecksCropSizePositive) {
TEST(SceneCropperTest, InitializesRetargeterChecksCropSizeNotExceedFrameSize) {
auto scene_summary = GetDefaultSceneKeyFrameCropSummary();
scene_summary.set_crop_window_height(kSceneHeight + 1);
SceneCropper scene_cropper;
CameraMotionOptions options;
options.mutable_polynomial_path_solver()->set_prior_frame_buffer_size(30);
SceneCropper scene_cropper(options, kSceneWidth, kSceneHeight);
std::vector<cv::Mat> cropped_frames;
std::vector<cv::Rect> crop_from_locations;
const auto& scene_frames = GetDefaultSceneFrames();
const auto status = scene_cropper.CropFrames(
scene_summary, scene_frames.size(), scene_frames,
GetDefaultFocusPointFrames(), GetFocusPointFrames(0), 0, 0,
scene_summary, GetTimestamps(kNumSceneFrames),
GetIsKeyframe(kNumSceneFrames), scene_frames,
GetDefaultFocusPointFrames(), GetFocusPointFrames(0), 0, 0, false,
&crop_from_locations, &cropped_frames);
EXPECT_FALSE(status.ok());
EXPECT_THAT(status.ToString(),
@@ -134,13 +163,16 @@ TEST(SceneCropperTest, InitializesRetargeterChecksCropSizeNotExceedFrameSize) {
// Checks that CropFrames works when there are not any prior FocusPointFrames.
TEST(SceneCropperTest, CropFramesWorksWithoutPriorFocusPointFrames) {
SceneCropper scene_cropper;
CameraMotionOptions options;
options.mutable_polynomial_path_solver()->set_prior_frame_buffer_size(30);
SceneCropper scene_cropper(options, kSceneWidth, kSceneHeight);
std::vector<cv::Mat> cropped_frames;
std::vector<cv::Rect> crop_from_locations;
const auto& scene_frames = GetDefaultSceneFrames();
MP_ASSERT_OK(scene_cropper.CropFrames(
GetDefaultSceneKeyFrameCropSummary(), scene_frames.size(), scene_frames,
GetDefaultFocusPointFrames(), GetFocusPointFrames(0), 0, 0,
GetDefaultSceneKeyFrameCropSummary(), GetTimestamps(kNumSceneFrames),
GetIsKeyframe(kNumSceneFrames), scene_frames,
GetDefaultFocusPointFrames(), GetFocusPointFrames(0), 0, 0, false,
&crop_from_locations, &cropped_frames));
ASSERT_EQ(cropped_frames.size(), kNumSceneFrames);
for (int i = 0; i < kNumSceneFrames; ++i) {
@@ -151,13 +183,16 @@ TEST(SceneCropperTest, CropFramesWorksWithoutPriorFocusPointFrames) {
// Checks that CropFrames works when there are prior FocusPointFrames.
TEST(SceneCropperTest, CropFramesWorksWithPriorFocusPointFrames) {
SceneCropper scene_cropper;
CameraMotionOptions options;
options.mutable_polynomial_path_solver()->set_prior_frame_buffer_size(30);
SceneCropper scene_cropper(options, kSceneWidth, kSceneHeight);
std::vector<cv::Mat> cropped_frames;
std::vector<cv::Rect> crop_from_locations;
const auto& scene_frames = GetDefaultSceneFrames();
MP_EXPECT_OK(scene_cropper.CropFrames(
GetDefaultSceneKeyFrameCropSummary(), scene_frames.size(), scene_frames,
GetDefaultFocusPointFrames(), GetFocusPointFrames(3), 0, 0,
GetDefaultSceneKeyFrameCropSummary(), GetTimestamps(scene_frames.size()),
GetIsKeyframe(scene_frames.size()), scene_frames,
GetDefaultFocusPointFrames(), GetFocusPointFrames(3), 0, 0, false,
&crop_from_locations, &cropped_frames));
EXPECT_EQ(cropped_frames.size(), kNumSceneFrames);
for (int i = 0; i < kNumSceneFrames; ++i) {
mediapipe/examples/desktop/autoflip/quality/scene_cropping_viz.cc
+36
View File
@@ -133,6 +133,42 @@ const cv::Scalar kWhite = cv::Scalar(255.0, 255.0, 255.0); // others
return ::mediapipe::OkStatus();
}
namespace {
cv::Rect LimitBounds(const cv::Rect& rect, const int max_width,
const int max_height) {
cv::Rect result;
result.x = fmax(rect.x, 0);
result.y = fmax(rect.y, 0);
result.width =
result.x + rect.width >= max_width ? max_width - result.x : rect.width;
result.height = result.y + rect.height >= max_height ? max_height - result.y
: rect.height;
return result;
}
} // namespace
::mediapipe::Status DrawDetectionAndFramingWindow(
const std::vector<cv::Mat>& org_scene_frames,
const std::vector<cv::Rect>& crop_from_locations,
const ImageFormat::Format image_format, const float overlay_opacity,
std::vector<std::unique_ptr<ImageFrame>>* viz_frames) {
for (int i = 0; i < org_scene_frames.size(); i++) {
const auto& scene_frame = org_scene_frames[i];
auto viz_frame = absl::make_unique<ImageFrame>(
image_format, scene_frame.cols, scene_frame.rows);
cv::Mat darkened = formats::MatView(viz_frame.get());
scene_frame.copyTo(darkened);
cv::Mat overlay = cv::Mat::zeros(darkened.size(), darkened.type());
cv::addWeighted(overlay, overlay_opacity, darkened, 1 - overlay_opacity, 0,
darkened);
const auto& crop_from_bounded =
LimitBounds(crop_from_locations[i], scene_frame.cols, scene_frame.rows);
scene_frame(crop_from_bounded).copyTo(darkened(crop_from_bounded));
viz_frames->push_back(std::move(viz_frame));
}
return ::mediapipe::OkStatus();
}
::mediapipe::Status DrawFocusPointAndCropWindow(
const std::vector<cv::Mat>& scene_frames,
const std::vector<FocusPointFrame>& focus_point_frames,
mediapipe/examples/desktop/autoflip/quality/scene_cropping_viz.h
+8
View File
@@ -55,6 +55,14 @@ namespace autoflip {
const mediapipe::ImageFormat::Format image_format,
std::vector<std::unique_ptr<ImageFrame>>* viz_frames);
// Draws the final smoothed path of the camera retargeter by darkening the
// removed areas.
::mediapipe::Status DrawDetectionAndFramingWindow(
const std::vector<cv::Mat>& org_scene_frames,
const std::vector<cv::Rect>& crop_from_locations,
const ImageFormat::Format image_format, const float overlay_opacity,
std::vector<std::unique_ptr<ImageFrame>>* viz_frames);
} // namespace autoflip
} // namespace mediapipe
mediapipe/examples/desktop/autoflip/quality/utils.cc
+2 -7
View File
@@ -203,7 +203,6 @@ void RectUnion(const Rect& rect_to_add, Rect* rect) {
}
::mediapipe::Status AggregateKeyFrameResults(
const std::vector<KeyFrameInfo>& key_frame_infos,
const KeyFrameCropOptions& key_frame_crop_options,
const std::vector<KeyFrameCropResult>& key_frame_crop_results,
const int scene_frame_width, const int scene_frame_height,
@@ -211,11 +210,7 @@ void RectUnion(const Rect& rect_to_add, Rect* rect) {
RET_CHECK_NE(scene_summary, nullptr)
<< "Output SceneKeyFrameCropSummary is null.";
const int num_key_frames = key_frame_infos.size();
RET_CHECK_EQ(num_key_frames, key_frame_crop_results.size())
<< "Inconsistent number of key frames:"
<< " num_key_frames = " << num_key_frames
<< " key_frame_crop_results.size() = " << key_frame_crop_results.size();
const int num_key_frames = key_frame_crop_results.size();
RET_CHECK_GT(scene_frame_width, 0) << "Non-positive frame width.";
RET_CHECK_GT(scene_frame_height, 0) << "Non-positive frame height.";
@@ -255,8 +250,8 @@ void RectUnion(const Rect& rect_to_add, Rect* rect) {
std::unique_ptr<Rect> required_crop_region_union = nullptr;
for (int i = 0; i < num_key_frames; ++i) {
auto* key_frame_compact_info = scene_summary->add_key_frame_compact_infos();
key_frame_compact_info->set_timestamp_ms(key_frame_infos[i].timestamp_ms());
const auto& result = key_frame_crop_results[i];
key_frame_compact_info->set_timestamp_ms(result.timestamp_ms());
if (result.are_required_regions_covered_in_target_size()) {
num_success_frames++;
}
mediapipe/examples/desktop/autoflip/quality/utils.h
-1
View File
@@ -54,7 +54,6 @@ namespace autoflip {
// Aggregates information from KeyFrameInfos and KeyFrameCropResults into
// SceneKeyFrameCropSummary.
::mediapipe::Status AggregateKeyFrameResults(
const std::vector<KeyFrameInfo>& key_frame_infos,
const KeyFrameCropOptions& key_frame_crop_options,
const std::vector<KeyFrameCropResult>& key_frame_crop_results,
const int scene_frame_width, const int scene_frame_height,
mediapipe/examples/desktop/autoflip/quality/utils_test.cc
+57 -90
View File
@@ -113,16 +113,6 @@ void AddDetectionFromScoreAndIsRequired(const double score,
detection->set_is_required(is_required);
}
// Returns default values for KeyFrameInfos. Populates timestamps using the
// default spacing kKeyFrameTimestampDiff starting from 0.
std::vector<KeyFrameInfo> GetDefaultKeyFrameInfos() {
std::vector<KeyFrameInfo> key_frame_infos(kNumKeyFrames);
for (int i = 0; i < kNumKeyFrames; ++i) {
key_frame_infos[i].set_timestamp_ms(kKeyFrameTimestampDiff * i);
}
return key_frame_infos;
}
// Returns default settings for KeyFrameCropOptions. Populates target size to be
// the default target size.
KeyFrameCropOptions GetDefaultKeyFrameCropOptions() {
@@ -148,6 +138,7 @@ std::vector<KeyFrameCropResult> GetDefaultKeyFrameCropResults() {
*(key_frame_crop_results[i].mutable_required_region()) =
MakeRect(10, 10, 20, 20);
key_frame_crop_results[i].set_region_score(1.0);
key_frame_crop_results[i].set_timestamp_ms(kKeyFrameTimestampDiff * i);
}
return key_frame_crop_results;
}
@@ -502,9 +493,8 @@ TEST(UtilTest, SetKeyFrameCropTargetSetsTargetSizeCorrectly) {
// Checks that AggregateKeyFrameResults checks output pointer is not null.
TEST(UtilTest, AggregateKeyFrameResultsChecksOutputNotNull) {
const auto status = AggregateKeyFrameResults(
GetDefaultKeyFrameInfos(), GetDefaultKeyFrameCropOptions(),
GetDefaultKeyFrameCropResults(), kOriginalWidth, kOriginalHeight,
nullptr);
GetDefaultKeyFrameCropOptions(), GetDefaultKeyFrameCropResults(),
kOriginalWidth, kOriginalHeight, nullptr);
EXPECT_FALSE(status.ok());
EXPECT_THAT(status.ToString(),
HasSubstr("Output SceneKeyFrameCropSummary is null."));
@@ -512,36 +502,20 @@ TEST(UtilTest, AggregateKeyFrameResultsChecksOutputNotNull) {
// Checks that AggregateKeyFrameResults handles the case of no key frames.
TEST(UtilTest, AggregateKeyFrameResultsHandlesNoKeyFrames) {
std::vector<KeyFrameInfo> key_frame_infos(0);
std::vector<KeyFrameCropResult> key_frame_crop_results(0);
SceneKeyFrameCropSummary scene_summary;
MP_EXPECT_OK(AggregateKeyFrameResults(
key_frame_infos, GetDefaultKeyFrameCropOptions(), key_frame_crop_results,
kOriginalWidth, kOriginalHeight, &scene_summary));
}
// Checks that AggregateKeyFrameResults checks that number of key frames is
// consistent between KeyFrameInfos and KeyFrameCropResults.
TEST(UtilTest, AggregateKeyFrameResultsChecksNumKeyFramesConsistent) {
std::vector<KeyFrameInfo> key_frame_infos(kNumKeyFrames);
std::vector<KeyFrameCropResult> key_frame_crop_results(kNumKeyFrames + 1);
SceneKeyFrameCropSummary scene_summary;
const auto status = AggregateKeyFrameResults(
key_frame_infos, GetDefaultKeyFrameCropOptions(), key_frame_crop_results,
kOriginalWidth, kOriginalHeight, &scene_summary);
EXPECT_FALSE(status.ok());
EXPECT_THAT(status.ToString(),
HasSubstr("Inconsistent number of key frames"));
MP_EXPECT_OK(AggregateKeyFrameResults(GetDefaultKeyFrameCropOptions(),
key_frame_crop_results, kOriginalWidth,
kOriginalHeight, &scene_summary));
}
// Checks that AggregateKeyFrameResults checks that frame size is valid.
TEST(UtilTest, AggregateKeyFrameResultsChecksFrameSizeValid) {
SceneKeyFrameCropSummary scene_summary;
const auto status = AggregateKeyFrameResults(
GetDefaultKeyFrameInfos(), GetDefaultKeyFrameCropOptions(),
GetDefaultKeyFrameCropResults(), kOriginalWidth, 0, &scene_summary);
GetDefaultKeyFrameCropOptions(), GetDefaultKeyFrameCropResults(),
kOriginalWidth, 0, &scene_summary);
EXPECT_FALSE(status.ok());
EXPECT_THAT(status.ToString(), HasSubstr("Non-positive frame height."));
}
@@ -553,9 +527,8 @@ TEST(UtilTest, AggregateKeyFrameResultsChecksTargetSizeValid) {
SceneKeyFrameCropSummary scene_summary;
const auto status = AggregateKeyFrameResults(
GetDefaultKeyFrameInfos(), key_frame_crop_options,
GetDefaultKeyFrameCropResults(), kOriginalWidth, kOriginalHeight,
&scene_summary);
key_frame_crop_options, GetDefaultKeyFrameCropResults(), kOriginalWidth,
kOriginalHeight, &scene_summary);
EXPECT_FALSE(status.ok());
EXPECT_THAT(status.ToString(), HasSubstr("Non-positive target width."));
}
@@ -568,9 +541,8 @@ TEST(UtilTest, AggregateKeyFrameResultsChecksTargetSizeNotExceedFrameSize) {
SceneKeyFrameCropSummary scene_summary;
const auto status = AggregateKeyFrameResults(
GetDefaultKeyFrameInfos(), key_frame_crop_options,
GetDefaultKeyFrameCropResults(), kOriginalWidth, kOriginalHeight,
&scene_summary);
key_frame_crop_options, GetDefaultKeyFrameCropResults(), kOriginalWidth,
kOriginalHeight, &scene_summary);
EXPECT_FALSE(status.ok());
EXPECT_THAT(status.ToString(),
HasSubstr("Target width exceeds frame width."));
@@ -578,19 +550,19 @@ TEST(UtilTest, AggregateKeyFrameResultsChecksTargetSizeNotExceedFrameSize) {
// Checks that AggregateKeyFrameResults packs KeyFrameCompactInfos.
TEST(UtilTest, AggregateKeyFrameResultsPacksKeyFrameCompactInfos) {
const auto key_frame_infos = GetDefaultKeyFrameInfos();
const auto key_frame_crop_results = GetDefaultKeyFrameCropResults();
SceneKeyFrameCropSummary scene_summary;
MP_EXPECT_OK(AggregateKeyFrameResults(
key_frame_infos, GetDefaultKeyFrameCropOptions(), key_frame_crop_results,
kOriginalWidth, kOriginalHeight, &scene_summary));
MP_EXPECT_OK(AggregateKeyFrameResults(GetDefaultKeyFrameCropOptions(),
key_frame_crop_results, kOriginalWidth,
kOriginalHeight, &scene_summary));
EXPECT_EQ(scene_summary.num_key_frames(), kNumKeyFrames);
EXPECT_EQ(scene_summary.key_frame_compact_infos_size(), kNumKeyFrames);
for (int i = 0; i < kNumKeyFrames; ++i) {
const auto& compact_info = scene_summary.key_frame_compact_infos(i);
EXPECT_EQ(compact_info.timestamp_ms(), key_frame_infos[i].timestamp_ms());
EXPECT_EQ(compact_info.timestamp_ms(),
key_frame_crop_results[i].timestamp_ms());
const auto center = RectCenter(key_frame_crop_results[i].region());
EXPECT_FLOAT_EQ(compact_info.center_x(), center.first);
EXPECT_FLOAT_EQ(compact_info.center_y(), center.second);
@@ -602,7 +574,6 @@ TEST(UtilTest, AggregateKeyFrameResultsPacksKeyFrameCompactInfos) {
// Checks that AggregateKeyFrameResults ensures the centered region of target
// size fits in frame bound.
TEST(UtilTest, AggregateKeyFrameResultsEnsuresCropRegionFitsInFrame) {
std::vector<KeyFrameInfo> key_frame_infos(1);
std::vector<KeyFrameCropResult> key_frame_crop_results(1);
auto* crop_region = key_frame_crop_results[0].mutable_region();
crop_region->set_x(0);
@@ -611,9 +582,9 @@ TEST(UtilTest, AggregateKeyFrameResultsEnsuresCropRegionFitsInFrame) {
crop_region->set_height(10);
SceneKeyFrameCropSummary scene_summary;
MP_EXPECT_OK(AggregateKeyFrameResults(
key_frame_infos, GetDefaultKeyFrameCropOptions(), key_frame_crop_results,
kOriginalWidth, kOriginalHeight, &scene_summary));
MP_EXPECT_OK(AggregateKeyFrameResults(GetDefaultKeyFrameCropOptions(),
key_frame_crop_results, kOriginalWidth,
kOriginalHeight, &scene_summary));
EXPECT_EQ(scene_summary.crop_window_width(), kTargetWidth);
EXPECT_EQ(scene_summary.crop_window_height(), kTargetHeight);
@@ -638,14 +609,13 @@ TEST(UtilTest, AggregateKeyFrameResultsEnsuresCropRegionFitsInFrame) {
// frames with empty regions.
TEST(UtilTest,
AggregateKeyFrameResultsSetsMinusOneForKeyFramesWithEmptyRegions) {
std::vector<KeyFrameInfo> key_frame_infos(1);
std::vector<KeyFrameCropResult> key_frame_crop_results(1);
key_frame_crop_results[0].set_region_is_empty(true);
SceneKeyFrameCropSummary scene_summary;
MP_EXPECT_OK(AggregateKeyFrameResults(
key_frame_infos, GetDefaultKeyFrameCropOptions(), key_frame_crop_results,
kOriginalWidth, kOriginalHeight, &scene_summary));
MP_EXPECT_OK(AggregateKeyFrameResults(GetDefaultKeyFrameCropOptions(),
key_frame_crop_results, kOriginalWidth,
kOriginalHeight, &scene_summary));
const auto& compact_info = scene_summary.key_frame_compact_infos(0);
EXPECT_FLOAT_EQ(compact_info.center_x(), -1.0f);
@@ -661,8 +631,8 @@ TEST(UtilTest, AggregateKeyFrameResultsRejectsNegativeCenter) {
SceneKeyFrameCropSummary scene_summary;
const auto status = AggregateKeyFrameResults(
GetDefaultKeyFrameInfos(), GetDefaultKeyFrameCropOptions(),
key_frame_crop_results, kOriginalWidth, kOriginalHeight, &scene_summary);
GetDefaultKeyFrameCropOptions(), key_frame_crop_results, kOriginalWidth,
kOriginalHeight, &scene_summary);
EXPECT_FALSE(status.ok());
EXPECT_THAT(status.ToString(), HasSubstr("Negative vertical center."));
}
@@ -674,8 +644,8 @@ TEST(UtilTest, AggregateKeyFrameResultsRejectsNegativeScore) {
SceneKeyFrameCropSummary scene_summary;
const auto status = AggregateKeyFrameResults(
GetDefaultKeyFrameInfos(), GetDefaultKeyFrameCropOptions(),
key_frame_crop_results, kOriginalWidth, kOriginalHeight, &scene_summary);
GetDefaultKeyFrameCropOptions(), key_frame_crop_results, kOriginalWidth,
kOriginalHeight, &scene_summary);
EXPECT_FALSE(status.ok());
EXPECT_THAT(status.ToString(), HasSubstr("Negative score."));
}
@@ -693,9 +663,9 @@ TEST(UtilTest, AggregateKeyFrameResultsSetsCenterRanges) {
}
SceneKeyFrameCropSummary scene_summary;
MP_EXPECT_OK(AggregateKeyFrameResults(
GetDefaultKeyFrameInfos(), GetDefaultKeyFrameCropOptions(),
key_frame_crop_results, kOriginalWidth, kOriginalHeight, &scene_summary));
MP_EXPECT_OK(AggregateKeyFrameResults(GetDefaultKeyFrameCropOptions(),
key_frame_crop_results, kOriginalWidth,
kOriginalHeight, &scene_summary));
EXPECT_FLOAT_EQ(scene_summary.key_frame_center_min_x(), 25.0f);
EXPECT_FLOAT_EQ(scene_summary.key_frame_center_max_x(), 45.0f);
@@ -712,9 +682,9 @@ TEST(UtilTest, AggregateKeyFrameResultsSetsScoreRange) {
}
SceneKeyFrameCropSummary scene_summary;
MP_EXPECT_OK(AggregateKeyFrameResults(
GetDefaultKeyFrameInfos(), GetDefaultKeyFrameCropOptions(),
key_frame_crop_results, kOriginalWidth, kOriginalHeight, &scene_summary));
MP_EXPECT_OK(AggregateKeyFrameResults(GetDefaultKeyFrameCropOptions(),
key_frame_crop_results, kOriginalWidth,
kOriginalHeight, &scene_summary));
EXPECT_FLOAT_EQ(scene_summary.key_frame_min_score(),
*std::min_element(scores.begin(), scores.end()));
@@ -727,9 +697,8 @@ TEST(UtilTest, AggregateKeyFrameResultsSetsScoreRange) {
TEST(UtilTest, AggregateKeyFrameResultsSetsCropWindowSizeToTargetSize) {
SceneKeyFrameCropSummary scene_summary;
MP_EXPECT_OK(AggregateKeyFrameResults(
GetDefaultKeyFrameInfos(), GetDefaultKeyFrameCropOptions(),
GetDefaultKeyFrameCropResults(), kOriginalWidth, kOriginalHeight,
&scene_summary));
GetDefaultKeyFrameCropOptions(), GetDefaultKeyFrameCropResults(),
kOriginalWidth, kOriginalHeight, &scene_summary));
EXPECT_EQ(scene_summary.crop_window_width(), kTargetWidth);
EXPECT_EQ(scene_summary.crop_window_height(), kTargetHeight);
}
@@ -741,9 +710,9 @@ TEST(UtilTest, AggregateKeyFrameResultsSetsCropWindowSizeExceedingTargetSize) {
key_frame_crop_results[0].mutable_region()->set_width(kTargetWidth + 1);
SceneKeyFrameCropSummary scene_summary;
MP_EXPECT_OK(AggregateKeyFrameResults(
GetDefaultKeyFrameInfos(), GetDefaultKeyFrameCropOptions(),
key_frame_crop_results, kOriginalWidth, kOriginalHeight, &scene_summary));
MP_EXPECT_OK(AggregateKeyFrameResults(GetDefaultKeyFrameCropOptions(),
key_frame_crop_results, kOriginalWidth,
kOriginalHeight, &scene_summary));
EXPECT_EQ(scene_summary.crop_window_width(), kTargetWidth + 1);
EXPECT_EQ(scene_summary.crop_window_height(), kTargetHeight);
}
@@ -753,9 +722,8 @@ TEST(UtilTest, AggregateKeyFrameResultsSetsCropWindowSizeExceedingTargetSize) {
TEST(UtilTest, AggregateKeyFrameResultsSetsHasSalientRegionTrue) {
SceneKeyFrameCropSummary scene_summary;
MP_EXPECT_OK(AggregateKeyFrameResults(
GetDefaultKeyFrameInfos(), GetDefaultKeyFrameCropOptions(),
GetDefaultKeyFrameCropResults(), kOriginalWidth, kOriginalHeight,
&scene_summary));
GetDefaultKeyFrameCropOptions(), GetDefaultKeyFrameCropResults(),
kOriginalWidth, kOriginalHeight, &scene_summary));
EXPECT_TRUE(scene_summary.has_salient_region());
}
@@ -768,9 +736,9 @@ TEST(UtilTest, AggregateKeyFrameResultsSetsHasSalientRegionFalse) {
}
SceneKeyFrameCropSummary scene_summary;
MP_EXPECT_OK(AggregateKeyFrameResults(
GetDefaultKeyFrameInfos(), GetDefaultKeyFrameCropOptions(),
key_frame_crop_results, kOriginalWidth, kOriginalHeight, &scene_summary));
MP_EXPECT_OK(AggregateKeyFrameResults(GetDefaultKeyFrameCropOptions(),
key_frame_crop_results, kOriginalWidth,
kOriginalHeight, &scene_summary));
EXPECT_FALSE(scene_summary.has_salient_region());
}
@@ -779,9 +747,8 @@ TEST(UtilTest, AggregateKeyFrameResultsSetsHasSalientRegionFalse) {
TEST(UtilTest, AggregateKeyFrameResultsSetsHasRequiredSalientRegionTrue) {
SceneKeyFrameCropSummary scene_summary;
MP_EXPECT_OK(AggregateKeyFrameResults(
GetDefaultKeyFrameInfos(), GetDefaultKeyFrameCropOptions(),
GetDefaultKeyFrameCropResults(), kOriginalWidth, kOriginalHeight,
&scene_summary));
GetDefaultKeyFrameCropOptions(), GetDefaultKeyFrameCropResults(),
kOriginalWidth, kOriginalHeight, &scene_summary));
EXPECT_TRUE(scene_summary.has_required_salient_region());
}
@@ -794,9 +761,9 @@ TEST(UtilTest, AggregateKeyFrameResultsSetsHasRequiredSalientRegionFalse) {
}
SceneKeyFrameCropSummary scene_summary;
MP_EXPECT_OK(AggregateKeyFrameResults(
GetDefaultKeyFrameInfos(), GetDefaultKeyFrameCropOptions(),
key_frame_crop_results, kOriginalWidth, kOriginalHeight, &scene_summary));
MP_EXPECT_OK(AggregateKeyFrameResults(GetDefaultKeyFrameCropOptions(),
key_frame_crop_results, kOriginalWidth,
kOriginalHeight, &scene_summary));
EXPECT_FALSE(scene_summary.has_required_salient_region());
}
@@ -810,9 +777,9 @@ TEST(UtilTest, AggregateKeyFrameResultsSetsKeyFrameRequiredCropRegionUnion) {
}
SceneKeyFrameCropSummary scene_summary;
MP_EXPECT_OK(AggregateKeyFrameResults(
GetDefaultKeyFrameInfos(), GetDefaultKeyFrameCropOptions(),
key_frame_crop_results, kOriginalWidth, kOriginalHeight, &scene_summary));
MP_EXPECT_OK(AggregateKeyFrameResults(GetDefaultKeyFrameCropOptions(),
key_frame_crop_results, kOriginalWidth,
kOriginalHeight, &scene_summary));
const auto& required_crop_region_union =
scene_summary.key_frame_required_crop_region_union();
EXPECT_EQ(required_crop_region_union.x(), 0);
@@ -832,9 +799,9 @@ TEST(UtilTest, AggregateKeyFrameResultsSetsFrameSuccessRate) {
}
SceneKeyFrameCropSummary scene_summary;
MP_EXPECT_OK(AggregateKeyFrameResults(
GetDefaultKeyFrameInfos(), GetDefaultKeyFrameCropOptions(),
key_frame_crop_results, kOriginalWidth, kOriginalHeight, &scene_summary));
MP_EXPECT_OK(AggregateKeyFrameResults(GetDefaultKeyFrameCropOptions(),
key_frame_crop_results, kOriginalWidth,
kOriginalHeight, &scene_summary));
EXPECT_FLOAT_EQ(scene_summary.frame_success_rate(), success_rate);
}
@@ -853,9 +820,9 @@ TEST(UtilTest, AggregateKeyFrameResultsSetsMotion) {
}
SceneKeyFrameCropSummary scene_summary;
MP_EXPECT_OK(AggregateKeyFrameResults(
GetDefaultKeyFrameInfos(), GetDefaultKeyFrameCropOptions(),
key_frame_crop_results, kOriginalWidth, kOriginalHeight, &scene_summary));
MP_EXPECT_OK(AggregateKeyFrameResults(GetDefaultKeyFrameCropOptions(),
key_frame_crop_results, kOriginalWidth,
kOriginalHeight, &scene_summary));
EXPECT_FLOAT_EQ(scene_summary.horizontal_motion_amount(), motion_x);
EXPECT_FLOAT_EQ(scene_summary.vertical_motion_amount(), motion_y);
}
mediapipe/examples/desktop/autoflip/subgraph/BUILD
+17
View File
@@ -28,6 +28,23 @@ mediapipe_simple_subgraph(
],
)
mediapipe_simple_subgraph(
name = "autoflip_front_face_detection_subgraph",
graph = "front_face_detection_subgraph.pbtxt",
register_as = "AutoFlipFrontFaceDetectionSubgraph",
visibility = ["//visibility:public"],
deps = [
"//mediapipe/calculators/image:image_transformation_calculator",
"//mediapipe/calculators/tflite:ssd_anchors_calculator",
"//mediapipe/calculators/tflite:tflite_converter_calculator",
"//mediapipe/calculators/tflite:tflite_inference_calculator",
"//mediapipe/calculators/tflite:tflite_tensors_to_detections_calculator",
"//mediapipe/calculators/util:detection_label_id_to_text_calculator",
"//mediapipe/calculators/util:detection_letterbox_removal_calculator",
"//mediapipe/calculators/util:non_max_suppression_calculator",
],
)
mediapipe_simple_subgraph(
name = "autoflip_object_detection_subgraph",
graph = "autoflip_object_detection_subgraph.pbtxt",
mediapipe/examples/desktop/autoflip/subgraph/front_face_detection_subgraph.pbtxt
+135
View File
@@ -0,0 +1,135 @@
# MediaPipe graph that performs face detection with TensorFlow Lite on CPU. Model paths setup for web use.
# TODO: parameterize input paths to support desktop use.
input_stream: "VIDEO:input_video"
output_stream: "DETECTIONS:output_detections"
# Transforms the input image on CPU to a 128x128 image. To scale the input
# image, the scale_mode option is set to FIT to preserve the aspect ratio,
# resulting in potential letterboxing in the transformed image.
node: {
calculator: "ImageTransformationCalculator"
input_stream: "IMAGE:input_video"
output_stream: "IMAGE:transformed_input_video_cpu"
output_stream: "LETTERBOX_PADDING:letterbox_padding"
options: {
[mediapipe.ImageTransformationCalculatorOptions.ext] {
output_width: 128
output_height: 128
scale_mode: FIT
}
}
}
# Converts the transformed input image on CPU into an image tensor stored as a
# TfLiteTensor.
node {
calculator: "TfLiteConverterCalculator"
input_stream: "IMAGE:transformed_input_video_cpu"
output_stream: "TENSORS:image_tensor"
}
# Runs a TensorFlow Lite model on CPU that takes an image tensor and outputs a
# vector of tensors representing, for instance, detection boxes/keypoints and
# scores.
node {
calculator: "TfLiteInferenceCalculator"
input_stream: "TENSORS:image_tensor"
output_stream: "TENSORS:detection_tensors"
options: {
[mediapipe.TfLiteInferenceCalculatorOptions.ext] {
model_path: "face_detection_front.tflite"
}
}
}
# Generates a single side packet containing a vector of SSD anchors based on
# the specification in the options.
node {
calculator: "SsdAnchorsCalculator"
output_side_packet: "anchors"
options: {
[mediapipe.SsdAnchorsCalculatorOptions.ext] {
num_layers: 4
min_scale: 0.1484375
max_scale: 0.75
input_size_height: 128
input_size_width: 128
anchor_offset_x: 0.5
anchor_offset_y: 0.5
strides: 8
strides: 16
strides: 16
strides: 16
aspect_ratios: 1.0
fixed_anchor_size: true
}
}
}
# Decodes the detection tensors generated by the TensorFlow Lite model, based on
# the SSD anchors and the specification in the options, into a vector of
# detections. Each detection describes a detected object.
node {
calculator: "TfLiteTensorsToDetectionsCalculator"
input_stream: "TENSORS:detection_tensors"
input_side_packet: "ANCHORS:anchors"
output_stream: "DETECTIONS:detections"
options: {
[mediapipe.TfLiteTensorsToDetectionsCalculatorOptions.ext] {
num_classes: 1
num_boxes: 896
num_coords: 16
box_coord_offset: 0
keypoint_coord_offset: 4
num_keypoints: 6
num_values_per_keypoint: 2
sigmoid_score: true
score_clipping_thresh: 100.0
reverse_output_order: true
x_scale: 128.0
y_scale: 128.0
h_scale: 128.0
w_scale: 128.0
min_score_thresh: 0.75
}
}
}
# Performs non-max suppression to remove excessive detections.
node {
calculator: "NonMaxSuppressionCalculator"
input_stream: "detections"
output_stream: "filtered_detections"
options: {
[mediapipe.NonMaxSuppressionCalculatorOptions.ext] {
min_suppression_threshold: 0.3
overlap_type: INTERSECTION_OVER_UNION
algorithm: WEIGHTED
return_empty_detections: true
}
}
}
# Maps detection label IDs to the corresponding label text ("Face"). The label
# map is provided in the label_map_path option.
node {
calculator: "DetectionLabelIdToTextCalculator"
input_stream: "filtered_detections"
output_stream: "labeled_detections"
options: {
[mediapipe.DetectionLabelIdToTextCalculatorOptions.ext] {
label_map_path: "face_detection_front_labelmap.txt"
}
}
}
# Adjusts detection locations (already normalized to [0.f, 1.f]) on the
# letterboxed image (after image transformation with the FIT scale mode) to the
# corresponding locations on the same image with the letterbox removed (the
# input image to the graph before image transformation).
node {
calculator: "DetectionLetterboxRemovalCalculator"
input_stream: "DETECTIONS:labeled_detections"
input_stream: "LETTERBOX_PADDING:letterbox_padding"
output_stream: "DETECTIONS:output_detections"
}