talksik/mediapipe
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Project import generated by Copybara.

Browse Source 
GitOrigin-RevId: 1610e588e497817fae2d9a458093ab6a370e2972
This commit is contained in:
MediaPipe Team
2021-08-18 17:45:46 -07:00
committed by jqtang
parent b899d17f18
commit 710fb3de58
158 changed files with 10104 additions and 1568 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mediapipe/examples/desktop/autoflip/calculators/BUILD
+2
View File
@@ -17,8 +17,10 @@ load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library"
licenses(["notice"])
package(default_visibility = [
"//buzz/diffractor/mediapipe:__subpackages__",
"//mediapipe/examples:__subpackages__",
"//mediapipe/viz:__subpackages__",
"//mediapipe/web/solutions:__subpackages__",
])
cc_library(
mediapipe/examples/desktop/autoflip/calculators/border_detection_calculator_test.cc
+18 -15
View File
@@ -43,6 +43,9 @@ namespace mediapipe {
namespace autoflip {
namespace {
constexpr char kDetectedBordersTag[] = "DETECTED_BORDERS";
constexpr char kVideoTag[] = "VIDEO";
const char kConfig[] = R"(
calculator: "BorderDetectionCalculator"
input_stream: "VIDEO:camera_frames"
@@ -81,14 +84,14 @@ TEST(BorderDetectionCalculatorTest, NoBorderTest) {
ImageFormat::SRGB, kTestFrameWidth, kTestFrameHeight);
cv::Mat input_mat = mediapipe::formats::MatView(input_frame.get());
input_mat.setTo(cv::Scalar(0, 0, 0));
runner->MutableInputs()->Tag("VIDEO").packets.push_back(
runner->MutableInputs()->Tag(kVideoTag).packets.push_back(
Adopt(input_frame.release()).At(Timestamp::PostStream()));
// Run the calculator.
MP_ASSERT_OK(runner->Run());
const std::vector<Packet>& output_packets =
runner->Outputs().Tag("DETECTED_BORDERS").packets;
runner->Outputs().Tag(kDetectedBordersTag).packets;
ASSERT_EQ(1, output_packets.size());
const auto& static_features = output_packets[0].Get<StaticFeatures>();
ASSERT_EQ(0, static_features.border().size());
@@ -115,14 +118,14 @@ TEST(BorderDetectionCalculatorTest, TopBorderTest) {
cv::Mat sub_image =
input_mat(cv::Rect(0, 0, kTestFrameWidth, kTopBorderHeight));
sub_image.setTo(cv::Scalar(255, 0, 0));
runner->MutableInputs()->Tag("VIDEO").packets.push_back(
runner->MutableInputs()->Tag(kVideoTag).packets.push_back(
Adopt(input_frame.release()).At(Timestamp::PostStream()));
// Run the calculator.
MP_ASSERT_OK(runner->Run());
const std::vector<Packet>& output_packets =
runner->Outputs().Tag("DETECTED_BORDERS").packets;
runner->Outputs().Tag(kDetectedBordersTag).packets;
ASSERT_EQ(1, output_packets.size());
const auto& static_features = output_packets[0].Get<StaticFeatures>();
ASSERT_EQ(1, static_features.border().size());
@@ -155,14 +158,14 @@ TEST(BorderDetectionCalculatorTest, TopBorderPadTest) {
cv::Mat sub_image =
input_mat(cv::Rect(0, 0, kTestFrameWidth, kTopBorderHeight));
sub_image.setTo(cv::Scalar(255, 0, 0));
runner->MutableInputs()->Tag("VIDEO").packets.push_back(
runner->MutableInputs()->Tag(kVideoTag).packets.push_back(
Adopt(input_frame.release()).At(Timestamp::PostStream()));
// Run the calculator.
MP_ASSERT_OK(runner->Run());
const std::vector<Packet>& output_packets =
runner->Outputs().Tag("DETECTED_BORDERS").packets;
runner->Outputs().Tag(kDetectedBordersTag).packets;
ASSERT_EQ(1, output_packets.size());
const auto& static_features = output_packets[0].Get<StaticFeatures>();
ASSERT_EQ(1, static_features.border().size());
@@ -197,14 +200,14 @@ TEST(BorderDetectionCalculatorTest, BottomBorderTest) {
input_mat(cv::Rect(0, kTestFrameHeight - kBottomBorderHeight,
kTestFrameWidth, kBottomBorderHeight));
bottom_image.setTo(cv::Scalar(255, 0, 0));
runner->MutableInputs()->Tag("VIDEO").packets.push_back(
runner->MutableInputs()->Tag(kVideoTag).packets.push_back(
Adopt(input_frame.release()).At(Timestamp::PostStream()));
// Run the calculator.
MP_ASSERT_OK(runner->Run());
const std::vector<Packet>& output_packets =
runner->Outputs().Tag("DETECTED_BORDERS").packets;
runner->Outputs().Tag(kDetectedBordersTag).packets;
ASSERT_EQ(1, output_packets.size());
const auto& static_features = output_packets[0].Get<StaticFeatures>();
ASSERT_EQ(1, static_features.border().size());
@@ -238,14 +241,14 @@ TEST(BorderDetectionCalculatorTest, TopBottomBorderTest) {
input_mat(cv::Rect(0, kTestFrameHeight - kBottomBorderHeight,
kTestFrameWidth, kBottomBorderHeight));
bottom_image.setTo(cv::Scalar(255, 0, 0));
runner->MutableInputs()->Tag("VIDEO").packets.push_back(
runner->MutableInputs()->Tag(kVideoTag).packets.push_back(
Adopt(input_frame.release()).At(Timestamp::PostStream()));
// Run the calculator.
MP_ASSERT_OK(runner->Run());
const std::vector<Packet>& output_packets =
runner->Outputs().Tag("DETECTED_BORDERS").packets;
runner->Outputs().Tag(kDetectedBordersTag).packets;
ASSERT_EQ(1, output_packets.size());
const auto& static_features = output_packets[0].Get<StaticFeatures>();
ASSERT_EQ(2, static_features.border().size());
@@ -291,14 +294,14 @@ TEST(BorderDetectionCalculatorTest, TopBottomBorderTestAspect2) {
input_mat(cv::Rect(0, kTestFrameHeightTall - kBottomBorderHeight,
kTestFrameWidthTall, kBottomBorderHeight));
bottom_image.setTo(cv::Scalar(255, 0, 0));
runner->MutableInputs()->Tag("VIDEO").packets.push_back(
runner->MutableInputs()->Tag(kVideoTag).packets.push_back(
Adopt(input_frame.release()).At(Timestamp::PostStream()));
// Run the calculator.
MP_ASSERT_OK(runner->Run());
const std::vector<Packet>& output_packets =
runner->Outputs().Tag("DETECTED_BORDERS").packets;
runner->Outputs().Tag(kDetectedBordersTag).packets;
ASSERT_EQ(1, output_packets.size());
const auto& static_features = output_packets[0].Get<StaticFeatures>();
ASSERT_EQ(2, static_features.border().size());
@@ -352,14 +355,14 @@ TEST(BorderDetectionCalculatorTest, DominantColor) {
input_mat(cv::Rect(0, 0, kTestFrameWidth / 2 + 50, kTestFrameHeight / 2));
sub_image.setTo(cv::Scalar(255, 0, 0));
runner->MutableInputs()->Tag("VIDEO").packets.push_back(
runner->MutableInputs()->Tag(kVideoTag).packets.push_back(
Adopt(input_frame.release()).At(Timestamp::PostStream()));
// Run the calculator.
MP_ASSERT_OK(runner->Run());
const std::vector<Packet>& output_packets =
runner->Outputs().Tag("DETECTED_BORDERS").packets;
runner->Outputs().Tag(kDetectedBordersTag).packets;
ASSERT_EQ(1, output_packets.size());
const auto& static_features = output_packets[0].Get<StaticFeatures>();
ASSERT_EQ(0, static_features.border().size());
@@ -383,7 +386,7 @@ void BM_Large(benchmark::State& state) {
cv::Mat sub_image =
input_mat(cv::Rect(0, 0, kTestFrameLargeWidth, kTopBorderHeight));
sub_image.setTo(cv::Scalar(255, 0, 0));
runner->MutableInputs()->Tag("VIDEO").packets.push_back(
runner->MutableInputs()->Tag(kVideoTag).packets.push_back(
Adopt(input_frame.release()).At(Timestamp::PostStream()));
// Run the calculator.
mediapipe/examples/desktop/autoflip/calculators/content_zooming_calculator.cc
+85 -6
View File
@@ -31,7 +31,11 @@ constexpr char kVideoSize[] = "VIDEO_SIZE";
constexpr char kSalientRegions[] = "SALIENT_REGIONS";
constexpr char kDetections[] = "DETECTIONS";
constexpr char kDetectedBorders[] = "BORDERS";
// Crop location as abs rect discretized.
constexpr char kCropRect[] = "CROP_RECT";
// Crop location as normalized rect.
constexpr char kNormalizedCropRect[] = "NORMALIZED_CROP_RECT";
// Crop location without position smoothing.
constexpr char kFirstCropRect[] = "FIRST_CROP_RECT";
// Can be used to control whether an animated zoom should actually performed
// (configured through option us_to_first_rect). If provided, a non-zero integer
@@ -51,6 +55,8 @@ constexpr float kFieldOfView = 60;
// Used to save state on Close and load state on Open in a new graph.
// Can be used to preserve state between graphs.
constexpr char kStateCache[] = "STATE_CACHE";
// Tolerance for zooming out recentering.
constexpr float kPixelTolerance = 3;
namespace mediapipe {
namespace autoflip {
@@ -166,6 +172,9 @@ absl::Status ContentZoomingCalculator::GetContract(
if (cc->Outputs().HasTag(kCropRect)) {
cc->Outputs().Tag(kCropRect).Set<mediapipe::Rect>();
}
if (cc->Outputs().HasTag(kNormalizedCropRect)) {
cc->Outputs().Tag(kNormalizedCropRect).Set<mediapipe::NormalizedRect>();
}
if (cc->Outputs().HasTag(kFirstCropRect)) {
cc->Outputs().Tag(kFirstCropRect).Set<mediapipe::NormalizedRect>();
}
@@ -553,6 +562,16 @@ absl::Status ContentZoomingCalculator::Process(
cc->Outputs().Tag(kCropRect).Add(default_rect.release(),
Timestamp(cc->InputTimestamp()));
}
if (cc->Outputs().HasTag(kNormalizedCropRect)) {
auto default_rect = absl::make_unique<mediapipe::NormalizedRect>();
default_rect->set_x_center(0.5);
default_rect->set_y_center(0.5);
default_rect->set_width(1.0);
default_rect->set_height(1.0);
cc->Outputs()
.Tag(kNormalizedCropRect)
.Add(default_rect.release(), Timestamp(cc->InputTimestamp()));
}
// Also provide a first crop rect: in this case a zero-sized one.
if (cc->Outputs().HasTag(kFirstCropRect)) {
cc->Outputs()
@@ -634,9 +653,9 @@ absl::Status ContentZoomingCalculator::Process(
// Compute smoothed zoom camera path.
MP_RETURN_IF_ERROR(path_solver_zoom_->AddObservation(
height, cc->InputTimestamp().Microseconds()));
int path_height;
float path_height;
MP_RETURN_IF_ERROR(path_solver_zoom_->GetState(&path_height));
int path_width = path_height * target_aspect_;
float path_width = path_height * target_aspect_;
// Update pixel-per-degree value for pan/tilt.
int target_height;
@@ -652,11 +671,48 @@ absl::Status ContentZoomingCalculator::Process(
offset_x, cc->InputTimestamp().Microseconds()));
MP_RETURN_IF_ERROR(path_solver_tilt_->AddObservation(
offset_y, cc->InputTimestamp().Microseconds()));
int path_offset_x;
float path_offset_x;
MP_RETURN_IF_ERROR(path_solver_pan_->GetState(&path_offset_x));
int path_offset_y;
float path_offset_y;
MP_RETURN_IF_ERROR(path_solver_tilt_->GetState(&path_offset_y));
float delta_height;
MP_RETURN_IF_ERROR(path_solver_zoom_->GetDeltaState(&delta_height));
int delta_width = delta_height * target_aspect_;
// Smooth centering when zooming out.
float remaining_width = target_width - path_width;
int width_space = frame_width_ - target_width;
if (abs(path_offset_x - frame_width_ / 2) >
width_space / 2 + kPixelTolerance &&
remaining_width > kPixelTolerance) {
float required_width =
abs(path_offset_x - frame_width_ / 2) - width_space / 2;
if (path_offset_x < frame_width_ / 2) {
path_offset_x += delta_width * (required_width / remaining_width);
MP_RETURN_IF_ERROR(path_solver_pan_->SetState(path_offset_x));
} else {
path_offset_x -= delta_width * (required_width / remaining_width);
MP_RETURN_IF_ERROR(path_solver_pan_->SetState(path_offset_x));
}
}
float remaining_height = target_height - path_height;
int height_space = frame_height_ - target_height;
if (abs(path_offset_y - frame_height_ / 2) >
height_space / 2 + kPixelTolerance &&
remaining_height > kPixelTolerance) {
float required_height =
abs(path_offset_y - frame_height_ / 2) - height_space / 2;
if (path_offset_y < frame_height_ / 2) {
path_offset_y += delta_height * (required_height / remaining_height);
MP_RETURN_IF_ERROR(path_solver_tilt_->SetState(path_offset_y));
} else {
path_offset_y -= delta_height * (required_height / remaining_height);
MP_RETURN_IF_ERROR(path_solver_tilt_->SetState(path_offset_y));
}
}
// Prevent box from extending beyond the image after camera smoothing.
if (path_offset_y - ceil(path_height / 2.0) < 0) {
path_offset_y = ceil(path_height / 2.0);
@@ -705,7 +761,7 @@ absl::Status ContentZoomingCalculator::Process(
is_animating = IsAnimatingToFirstRect(cc->InputTimestamp());
}
// Transmit downstream to glcroppingcalculator.
// Transmit downstream to glcroppingcalculator in discrete int values.
if (cc->Outputs().HasTag(kCropRect)) {
std::unique_ptr<mediapipe::Rect> gpu_rect;
if (is_animating) {
@@ -716,13 +772,36 @@ absl::Status ContentZoomingCalculator::Process(
} else {
gpu_rect = absl::make_unique<mediapipe::Rect>();
gpu_rect->set_x_center(path_offset_x);
gpu_rect->set_width(path_height * target_aspect_);
gpu_rect->set_width(path_width);
gpu_rect->set_y_center(path_offset_y);
gpu_rect->set_height(path_height);
}
cc->Outputs().Tag(kCropRect).Add(gpu_rect.release(),
Timestamp(cc->InputTimestamp()));
}
if (cc->Outputs().HasTag(kNormalizedCropRect)) {
std::unique_ptr<mediapipe::NormalizedRect> gpu_rect =
absl::make_unique<mediapipe::NormalizedRect>();
float float_frame_width = static_cast<float>(frame_width_);
float float_frame_height = static_cast<float>(frame_height_);
if (is_animating) {
auto rect =
GetAnimationRect(frame_width, frame_height, cc->InputTimestamp());
MP_RETURN_IF_ERROR(rect.status());
gpu_rect->set_x_center(rect->x_center() / float_frame_width);
gpu_rect->set_width(rect->width() / float_frame_width);
gpu_rect->set_y_center(rect->y_center() / float_frame_height);
gpu_rect->set_height(rect->height() / float_frame_height);
} else {
gpu_rect->set_x_center(path_offset_x / float_frame_width);
gpu_rect->set_width(path_width / float_frame_width);
gpu_rect->set_y_center(path_offset_y / float_frame_height);
gpu_rect->set_height(path_height / float_frame_height);
}
cc->Outputs()
.Tag(kNormalizedCropRect)
.Add(gpu_rect.release(), Timestamp(cc->InputTimestamp()));
}
if (cc->Outputs().HasTag(kFirstCropRect)) {
cc->Outputs()
mediapipe/examples/desktop/autoflip/calculators/content_zooming_calculator_test.cc
+142 -93
View File
@@ -38,6 +38,17 @@ namespace mediapipe {
namespace autoflip {
namespace {
constexpr char kFirstCropRectTag[] = "FIRST_CROP_RECT";
constexpr char kStateCacheTag[] = "STATE_CACHE";
constexpr char kCropRectTag[] = "CROP_RECT";
constexpr char kBordersTag[] = "BORDERS";
constexpr char kSalientRegionsTag[] = "SALIENT_REGIONS";
constexpr char kVideoTag[] = "VIDEO";
constexpr char kMaxZoomFactorPctTag[] = "MAX_ZOOM_FACTOR_PCT";
constexpr char kAnimateZoomTag[] = "ANIMATE_ZOOM";
constexpr char kVideoSizeTag[] = "VIDEO_SIZE";
constexpr char kDetectionsTag[] = "DETECTIONS";
const char kConfigA[] = R"(
calculator: "ContentZoomingCalculator"
input_stream: "VIDEO:camera_frames"
@@ -48,12 +59,15 @@ const char kConfigA[] = R"(
max_zoom_value_deg: 0
kinematic_options_zoom {
min_motion_to_reframe: 1.2
max_velocity: 18
}
kinematic_options_tilt {
min_motion_to_reframe: 1.2
max_velocity: 18
}
kinematic_options_pan {
min_motion_to_reframe: 1.2
max_velocity: 18
}
}
}
@@ -73,12 +87,15 @@ const char kConfigB[] = R"(
max_zoom_value_deg: 0
kinematic_options_zoom {
min_motion_to_reframe: 1.2
max_velocity: 18
}
kinematic_options_tilt {
min_motion_to_reframe: 1.2
max_velocity: 18
}
kinematic_options_pan {
min_motion_to_reframe: 1.2
max_velocity: 18
}
}
}
@@ -94,12 +111,15 @@ const char kConfigC[] = R"(
max_zoom_value_deg: 0
kinematic_options_zoom {
min_motion_to_reframe: 1.2
max_velocity: 18
}
kinematic_options_tilt {
min_motion_to_reframe: 1.2
max_velocity: 18
}
kinematic_options_pan {
min_motion_to_reframe: 1.2
max_velocity: 18
}
}
}
@@ -111,17 +131,21 @@ const char kConfigD[] = R"(
input_stream: "DETECTIONS:detections"
output_stream: "CROP_RECT:rect"
output_stream: "FIRST_CROP_RECT:first_rect"
output_stream: "NORMALIZED_CROP_RECT:float_rect"
options: {
[mediapipe.autoflip.ContentZoomingCalculatorOptions.ext]: {
max_zoom_value_deg: 0
kinematic_options_zoom {
min_motion_to_reframe: 1.2
max_velocity: 18
}
kinematic_options_tilt {
min_motion_to_reframe: 1.2
max_velocity: 18
}
kinematic_options_pan {
min_motion_to_reframe: 1.2
max_velocity: 18
}
}
}
@@ -139,12 +163,15 @@ const char kConfigE[] = R"(
max_zoom_value_deg: 0
kinematic_options_zoom {
min_motion_to_reframe: 1.2
max_velocity: 18
}
kinematic_options_tilt {
min_motion_to_reframe: 1.2
max_velocity: 18
}
kinematic_options_pan {
min_motion_to_reframe: 1.2
max_velocity: 18
}
}
}
@@ -162,12 +189,15 @@ const char kConfigF[] = R"(
max_zoom_value_deg: 0
kinematic_options_zoom {
min_motion_to_reframe: 1.2
max_velocity: 18
}
kinematic_options_tilt {
min_motion_to_reframe: 1.2
max_velocity: 18
}
kinematic_options_pan {
min_motion_to_reframe: 1.2
max_velocity: 18
}
}
}
@@ -220,17 +250,17 @@ void AddDetectionFrameSize(const cv::Rect_<float>& position, const int64 time,
detections->push_back(detection);
}
runner->MutableInputs()
->Tag("DETECTIONS")
->Tag(kDetectionsTag)
.packets.push_back(Adopt(detections.release()).At(Timestamp(time)));
auto input_size = ::absl::make_unique<std::pair<int, int>>(width, height);
runner->MutableInputs()
->Tag("VIDEO_SIZE")
->Tag(kVideoSizeTag)
.packets.push_back(Adopt(input_size.release()).At(Timestamp(time)));
if (flags.animated_zoom.has_value()) {
runner->MutableInputs()
->Tag("ANIMATE_ZOOM")
->Tag(kAnimateZoomTag)
.packets.push_back(
mediapipe::MakePacket<bool>(flags.animated_zoom.value())
.At(Timestamp(time)));
@@ -238,7 +268,7 @@ void AddDetectionFrameSize(const cv::Rect_<float>& position, const int64 time,
if (flags.max_zoom_factor_percent.has_value()) {
runner->MutableInputs()
->Tag("MAX_ZOOM_FACTOR_PCT")
->Tag(kMaxZoomFactorPctTag)
.packets.push_back(
mediapipe::MakePacket<int>(flags.max_zoom_factor_percent.value())
.At(Timestamp(time)));
@@ -250,6 +280,21 @@ void AddDetection(const cv::Rect_<float>& position, const int64 time,
AddDetectionFrameSize(position, time, 1000, 1000, runner);
}
void CheckCropRectFloats(const float x_center, const float y_center,
const float width, const float height,
const int frame_number,
const CalculatorRunner::StreamContentsSet& output) {
ASSERT_GT(output.Tag("NORMALIZED_CROP_RECT").packets.size(), frame_number);
auto float_rect = output.Tag("NORMALIZED_CROP_RECT")
.packets[frame_number]
.Get<mediapipe::NormalizedRect>();
EXPECT_FLOAT_EQ(float_rect.x_center(), x_center);
EXPECT_FLOAT_EQ(float_rect.y_center(), y_center);
EXPECT_FLOAT_EQ(float_rect.width(), width);
EXPECT_FLOAT_EQ(float_rect.height(), height);
}
void CheckCropRect(const int x_center, const int y_center, const int width,
const int height, const int frame_number,
const std::vector<Packet>& output_packets) {
@@ -274,21 +319,21 @@ TEST(ContentZoomingCalculatorTest, ZoomTest) {
auto input_frame =
::absl::make_unique<ImageFrame>(ImageFormat::SRGB, 1000, 1000);
runner->MutableInputs()->Tag("VIDEO").packets.push_back(
runner->MutableInputs()->Tag(kVideoTag).packets.push_back(
Adopt(input_frame.release()).At(Timestamp(0)));
runner->MutableInputs()
->Tag("SALIENT_REGIONS")
->Tag(kSalientRegionsTag)
.packets.push_back(Adopt(detection_set.release()).At(Timestamp(0)));
// Run the calculator.
MP_ASSERT_OK(runner->Run());
const std::vector<Packet>& output_packets =
runner->Outputs().Tag("BORDERS").packets;
runner->Outputs().Tag(kBordersTag).packets;
ASSERT_EQ(1, output_packets.size());
const auto& static_features = output_packets[0].Get<StaticFeatures>();
CheckBorder(static_features, 1000, 1000, 495, 395);
CheckBorder(static_features, 1000, 1000, 494, 394);
}
TEST(ContentZoomingCalculatorTest, ZoomTestFullPTZ) {
@@ -297,7 +342,7 @@ TEST(ContentZoomingCalculatorTest, ZoomTestFullPTZ) {
AddDetection(cv::Rect_<float>(.4, .5, .1, .1), 0, runner.get());
MP_ASSERT_OK(runner->Run());
CheckCropRect(450, 550, 111, 111, 0,
runner->Outputs().Tag("CROP_RECT").packets);
runner->Outputs().Tag(kCropRectTag).packets);
}
TEST(ContentZoomingCalculatorTest, PanConfig) {
@@ -313,9 +358,9 @@ TEST(ContentZoomingCalculatorTest, PanConfig) {
AddDetection(cv::Rect_<float>(.45, .55, .15, .15), 1000000, runner.get());
MP_ASSERT_OK(runner->Run());
CheckCropRect(450, 550, 111, 111, 0,
runner->Outputs().Tag("CROP_RECT").packets);
runner->Outputs().Tag(kCropRectTag).packets);
CheckCropRect(483, 550, 111, 111, 1,
runner->Outputs().Tag("CROP_RECT").packets);
runner->Outputs().Tag(kCropRectTag).packets);
}
TEST(ContentZoomingCalculatorTest, PanConfigWithCache) {
@@ -330,31 +375,31 @@ TEST(ContentZoomingCalculatorTest, PanConfigWithCache) {
options->mutable_kinematic_options_zoom()->set_min_motion_to_reframe(50.0);
{
auto runner = ::absl::make_unique<CalculatorRunner>(config);
runner->MutableSidePackets()->Tag("STATE_CACHE") = MakePacket<
runner->MutableSidePackets()->Tag(kStateCacheTag) = MakePacket<
mediapipe::autoflip::ContentZoomingCalculatorStateCacheType*>(&cache);
AddDetection(cv::Rect_<float>(.4, .5, .1, .1), 0, runner.get());
MP_ASSERT_OK(runner->Run());
CheckCropRect(450, 550, 111, 111, 0,
runner->Outputs().Tag("CROP_RECT").packets);
runner->Outputs().Tag(kCropRectTag).packets);
}
{
auto runner = ::absl::make_unique<CalculatorRunner>(config);
runner->MutableSidePackets()->Tag("STATE_CACHE") = MakePacket<
runner->MutableSidePackets()->Tag(kStateCacheTag) = MakePacket<
mediapipe::autoflip::ContentZoomingCalculatorStateCacheType*>(&cache);
AddDetection(cv::Rect_<float>(.45, .55, .15, .15), 1000000, runner.get());
MP_ASSERT_OK(runner->Run());
CheckCropRect(483, 550, 111, 111, 0,
runner->Outputs().Tag("CROP_RECT").packets);
runner->Outputs().Tag(kCropRectTag).packets);
}
// Now repeat the last frame for a new runner without the cache to see a reset
{
auto runner = ::absl::make_unique<CalculatorRunner>(config);
runner->MutableSidePackets()->Tag("STATE_CACHE") = MakePacket<
runner->MutableSidePackets()->Tag(kStateCacheTag) = MakePacket<
mediapipe::autoflip::ContentZoomingCalculatorStateCacheType*>(nullptr);
AddDetection(cv::Rect_<float>(.45, .55, .15, .15), 2000000, runner.get());
MP_ASSERT_OK(runner->Run());
CheckCropRect(525, 625, 166, 166, 0, // Without a cache, state was lost.
runner->Outputs().Tag("CROP_RECT").packets);
runner->Outputs().Tag(kCropRectTag).packets);
}
}
@@ -371,9 +416,9 @@ TEST(ContentZoomingCalculatorTest, TiltConfig) {
AddDetection(cv::Rect_<float>(.45, .55, .15, .15), 1000000, runner.get());
MP_ASSERT_OK(runner->Run());
CheckCropRect(450, 550, 111, 111, 0,
runner->Outputs().Tag("CROP_RECT").packets);
runner->Outputs().Tag(kCropRectTag).packets);
CheckCropRect(450, 583, 111, 111, 1,
runner->Outputs().Tag("CROP_RECT").packets);
runner->Outputs().Tag(kCropRectTag).packets);
}
TEST(ContentZoomingCalculatorTest, ZoomConfig) {
@@ -389,9 +434,9 @@ TEST(ContentZoomingCalculatorTest, ZoomConfig) {
AddDetection(cv::Rect_<float>(.45, .55, .15, .15), 1000000, runner.get());
MP_ASSERT_OK(runner->Run());
CheckCropRect(450, 550, 111, 111, 0,
runner->Outputs().Tag("CROP_RECT").packets);
CheckCropRect(450, 550, 139, 139, 1,
runner->Outputs().Tag("CROP_RECT").packets);
runner->Outputs().Tag(kCropRectTag).packets);
CheckCropRect(450, 550, 138, 138, 1,
runner->Outputs().Tag(kCropRectTag).packets);
}
TEST(ContentZoomingCalculatorTest, ZoomConfigWithCache) {
@@ -406,31 +451,31 @@ TEST(ContentZoomingCalculatorTest, ZoomConfigWithCache) {
options->mutable_kinematic_options_zoom()->set_update_rate_seconds(2);
{
auto runner = ::absl::make_unique<CalculatorRunner>(config);
runner->MutableSidePackets()->Tag("STATE_CACHE") = MakePacket<
runner->MutableSidePackets()->Tag(kStateCacheTag) = MakePacket<
mediapipe::autoflip::ContentZoomingCalculatorStateCacheType*>(&cache);
AddDetection(cv::Rect_<float>(.4, .5, .1, .1), 0, runner.get());
MP_ASSERT_OK(runner->Run());
CheckCropRect(450, 550, 111, 111, 0,
runner->Outputs().Tag("CROP_RECT").packets);
runner->Outputs().Tag(kCropRectTag).packets);
}
{
auto runner = ::absl::make_unique<CalculatorRunner>(config);
runner->MutableSidePackets()->Tag("STATE_CACHE") = MakePacket<
runner->MutableSidePackets()->Tag(kStateCacheTag) = MakePacket<
mediapipe::autoflip::ContentZoomingCalculatorStateCacheType*>(&cache);
AddDetection(cv::Rect_<float>(.45, .55, .15, .15), 1000000, runner.get());
MP_ASSERT_OK(runner->Run());
CheckCropRect(450, 550, 139, 139, 0,
runner->Outputs().Tag("CROP_RECT").packets);
CheckCropRect(450, 550, 138, 138, 0,
runner->Outputs().Tag(kCropRectTag).packets);
}
// Now repeat the last frame for a new runner without the cache to see a reset
{
auto runner = ::absl::make_unique<CalculatorRunner>(config);
runner->MutableSidePackets()->Tag("STATE_CACHE") = MakePacket<
runner->MutableSidePackets()->Tag(kStateCacheTag) = MakePacket<
mediapipe::autoflip::ContentZoomingCalculatorStateCacheType*>(nullptr);
AddDetection(cv::Rect_<float>(.45, .55, .15, .15), 2000000, runner.get());
MP_ASSERT_OK(runner->Run());
CheckCropRect(525, 625, 166, 166, 0, // Without a cache, state was lost.
runner->Outputs().Tag("CROP_RECT").packets);
runner->Outputs().Tag(kCropRectTag).packets);
}
}
@@ -448,18 +493,18 @@ TEST(ContentZoomingCalculatorTest, MinAspectBorderValues) {
auto input_frame =
::absl::make_unique<ImageFrame>(ImageFormat::SRGB, 1000, 1000);
runner->MutableInputs()->Tag("VIDEO").packets.push_back(
runner->MutableInputs()->Tag(kVideoTag).packets.push_back(
Adopt(input_frame.release()).At(Timestamp(0)));
runner->MutableInputs()
->Tag("SALIENT_REGIONS")
->Tag(kSalientRegionsTag)
.packets.push_back(Adopt(detection_set.release()).At(Timestamp(0)));
// Run the calculator.
MP_ASSERT_OK(runner->Run());
const std::vector<Packet>& output_packets =
runner->Outputs().Tag("BORDERS").packets;
runner->Outputs().Tag(kBordersTag).packets;
ASSERT_EQ(1, output_packets.size());
const auto& static_features = output_packets[0].Get<StaticFeatures>();
CheckBorder(static_features, 1000, 1000, 250, 250);
@@ -485,18 +530,18 @@ TEST(ContentZoomingCalculatorTest, TwoFacesWide) {
auto input_frame =
::absl::make_unique<ImageFrame>(ImageFormat::SRGB, 1000, 1000);
runner->MutableInputs()->Tag("VIDEO").packets.push_back(
runner->MutableInputs()->Tag(kVideoTag).packets.push_back(
Adopt(input_frame.release()).At(Timestamp(0)));
runner->MutableInputs()
->Tag("SALIENT_REGIONS")
->Tag(kSalientRegionsTag)
.packets.push_back(Adopt(detection_set.release()).At(Timestamp(0)));
// Run the calculator.
MP_ASSERT_OK(runner->Run());
const std::vector<Packet>& output_packets =
runner->Outputs().Tag("BORDERS").packets;
runner->Outputs().Tag(kBordersTag).packets;
ASSERT_EQ(1, output_packets.size());
const auto& static_features = output_packets[0].Get<StaticFeatures>();
@@ -510,18 +555,18 @@ TEST(ContentZoomingCalculatorTest, NoDetectionOnInit) {
auto input_frame =
::absl::make_unique<ImageFrame>(ImageFormat::SRGB, 1000, 1000);
runner->MutableInputs()->Tag("VIDEO").packets.push_back(
runner->MutableInputs()->Tag(kVideoTag).packets.push_back(
Adopt(input_frame.release()).At(Timestamp(0)));
runner->MutableInputs()
->Tag("SALIENT_REGIONS")
->Tag(kSalientRegionsTag)
.packets.push_back(Adopt(detection_set.release()).At(Timestamp(0)));
// Run the calculator.
MP_ASSERT_OK(runner->Run());
const std::vector<Packet>& output_packets =
runner->Outputs().Tag("BORDERS").packets;
runner->Outputs().Tag(kBordersTag).packets;
ASSERT_EQ(1, output_packets.size());
const auto& static_features = output_packets[0].Get<StaticFeatures>();
@@ -542,21 +587,21 @@ TEST(ContentZoomingCalculatorTest, ZoomTestPairSize) {
auto input_size = ::absl::make_unique<std::pair<int, int>>(1000, 1000);
runner->MutableInputs()
->Tag("VIDEO_SIZE")
->Tag(kVideoSizeTag)
.packets.push_back(Adopt(input_size.release()).At(Timestamp(0)));
runner->MutableInputs()
->Tag("SALIENT_REGIONS")
->Tag(kSalientRegionsTag)
.packets.push_back(Adopt(detection_set.release()).At(Timestamp(0)));
// Run the calculator.
MP_ASSERT_OK(runner->Run());
const std::vector<Packet>& output_packets =
runner->Outputs().Tag("BORDERS").packets;
runner->Outputs().Tag(kBordersTag).packets;
ASSERT_EQ(1, output_packets.size());
const auto& static_features = output_packets[0].Get<StaticFeatures>();
CheckBorder(static_features, 1000, 1000, 495, 395);
CheckBorder(static_features, 1000, 1000, 494, 394);
}
TEST(ContentZoomingCalculatorTest, ZoomTestNearOutsideBorder) {
@@ -571,9 +616,9 @@ TEST(ContentZoomingCalculatorTest, ZoomTestNearOutsideBorder) {
AddDetection(cv::Rect_<float>(.9, .9, .1, .1), 1000000, runner.get());
MP_ASSERT_OK(runner->Run());
CheckCropRect(972, 972, 55, 55, 0,
runner->Outputs().Tag("CROP_RECT").packets);
CheckCropRect(958, 958, 83, 83, 1,
runner->Outputs().Tag("CROP_RECT").packets);
runner->Outputs().Tag(kCropRectTag).packets);
CheckCropRect(944, 944, 83, 83, 1,
runner->Outputs().Tag(kCropRectTag).packets);
}
TEST(ContentZoomingCalculatorTest, ZoomTestNearInsideBorder) {
@@ -587,8 +632,8 @@ TEST(ContentZoomingCalculatorTest, ZoomTestNearInsideBorder) {
AddDetection(cv::Rect_<float>(0, 0, .05, .05), 0, runner.get());
AddDetection(cv::Rect_<float>(0, 0, .1, .1), 1000000, runner.get());
MP_ASSERT_OK(runner->Run());
CheckCropRect(28, 28, 55, 55, 0, runner->Outputs().Tag("CROP_RECT").packets);
CheckCropRect(42, 42, 83, 83, 1, runner->Outputs().Tag("CROP_RECT").packets);
CheckCropRect(28, 28, 55, 55, 0, runner->Outputs().Tag(kCropRectTag).packets);
CheckCropRect(56, 56, 83, 83, 1, runner->Outputs().Tag(kCropRectTag).packets);
}
TEST(ContentZoomingCalculatorTest, VerticalShift) {
@@ -601,7 +646,9 @@ TEST(ContentZoomingCalculatorTest, VerticalShift) {
MP_ASSERT_OK(runner->Run());
// 1000px * .1 offset + 1000*.1*.1 shift = 170
CheckCropRect(150, 170, 111, 111, 0,
runner->Outputs().Tag("CROP_RECT").packets);
runner->Outputs().Tag(kCropRectTag).packets);
CheckCropRectFloats(150 / 1000.0, 170 / 1000.0, 111 / 1000.0, 111 / 1000.0, 0,
runner->Outputs());
}
TEST(ContentZoomingCalculatorTest, HorizontalShift) {
@@ -614,7 +661,9 @@ TEST(ContentZoomingCalculatorTest, HorizontalShift) {
MP_ASSERT_OK(runner->Run());
// 1000px * .1 offset + 1000*.1*.1 shift = 170
CheckCropRect(170, 150, 111, 111, 0,
runner->Outputs().Tag("CROP_RECT").packets);
runner->Outputs().Tag(kCropRectTag).packets);
CheckCropRectFloats(170 / 1000.0, 150 / 1000.0, 111 / 1000.0, 111 / 1000.0, 0,
runner->Outputs());
}
TEST(ContentZoomingCalculatorTest, ShiftOutsideBounds) {
@@ -627,14 +676,14 @@ TEST(ContentZoomingCalculatorTest, ShiftOutsideBounds) {
AddDetection(cv::Rect_<float>(.9, 0, .1, .1), 0, runner.get());
MP_ASSERT_OK(runner->Run());
CheckCropRect(944, 56, 111, 111, 0,
runner->Outputs().Tag("CROP_RECT").packets);
runner->Outputs().Tag(kCropRectTag).packets);
}
TEST(ContentZoomingCalculatorTest, EmptySize) {
auto config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfigD);
auto runner = ::absl::make_unique<CalculatorRunner>(config);
MP_ASSERT_OK(runner->Run());
ASSERT_EQ(runner->Outputs().Tag("CROP_RECT").packets.size(), 0);
ASSERT_EQ(runner->Outputs().Tag(kCropRectTag).packets.size(), 0);
}
TEST(ContentZoomingCalculatorTest, EmptyDetections) {
@@ -642,11 +691,11 @@ TEST(ContentZoomingCalculatorTest, EmptyDetections) {
auto runner = ::absl::make_unique<CalculatorRunner>(config);
auto input_size = ::absl::make_unique<std::pair<int, int>>(1000, 1000);
runner->MutableInputs()
->Tag("VIDEO_SIZE")
->Tag(kVideoSizeTag)
.packets.push_back(Adopt(input_size.release()).At(Timestamp(0)));
MP_ASSERT_OK(runner->Run());
CheckCropRect(500, 500, 1000, 1000, 0,
runner->Outputs().Tag("CROP_RECT").packets);
runner->Outputs().Tag(kCropRectTag).packets);
}
TEST(ContentZoomingCalculatorTest, ResolutionChangeStationary) {
@@ -658,9 +707,9 @@ TEST(ContentZoomingCalculatorTest, ResolutionChangeStationary) {
runner.get());
MP_ASSERT_OK(runner->Run());
CheckCropRect(500, 500, 222, 222, 0,
runner->Outputs().Tag("CROP_RECT").packets);
runner->Outputs().Tag(kCropRectTag).packets);
CheckCropRect(500 * 0.5, 500 * 0.5, 222 * 0.5, 222 * 0.5, 1,
runner->Outputs().Tag("CROP_RECT").packets);
runner->Outputs().Tag(kCropRectTag).packets);
}
TEST(ContentZoomingCalculatorTest, ResolutionChangeStationaryWithCache) {
@@ -669,23 +718,23 @@ TEST(ContentZoomingCalculatorTest, ResolutionChangeStationaryWithCache) {
config.add_input_side_packet("STATE_CACHE:state_cache");
{
auto runner = ::absl::make_unique<CalculatorRunner>(config);
runner->MutableSidePackets()->Tag("STATE_CACHE") = MakePacket<
runner->MutableSidePackets()->Tag(kStateCacheTag) = MakePacket<
mediapipe::autoflip::ContentZoomingCalculatorStateCacheType*>(&cache);
AddDetectionFrameSize(cv::Rect_<float>(.4, .4, .2, .2), 0, 1000, 1000,
runner.get());
MP_ASSERT_OK(runner->Run());
CheckCropRect(500, 500, 222, 222, 0,
runner->Outputs().Tag("CROP_RECT").packets);
runner->Outputs().Tag(kCropRectTag).packets);
}
{
auto runner = ::absl::make_unique<CalculatorRunner>(config);
runner->MutableSidePackets()->Tag("STATE_CACHE") = MakePacket<
runner->MutableSidePackets()->Tag(kStateCacheTag) = MakePacket<
mediapipe::autoflip::ContentZoomingCalculatorStateCacheType*>(&cache);
AddDetectionFrameSize(cv::Rect_<float>(.4, .4, .2, .2), 1, 500, 500,
runner.get());
MP_ASSERT_OK(runner->Run());
CheckCropRect(500 * 0.5, 500 * 0.5, 222 * 0.5, 222 * 0.5, 0,
runner->Outputs().Tag("CROP_RECT").packets);
runner->Outputs().Tag(kCropRectTag).packets);
}
}
@@ -700,11 +749,11 @@ TEST(ContentZoomingCalculatorTest, ResolutionChangeZooming) {
runner.get());
MP_ASSERT_OK(runner->Run());
CheckCropRect(500, 500, 888, 888, 0,
runner->Outputs().Tag("CROP_RECT").packets);
runner->Outputs().Tag(kCropRectTag).packets);
CheckCropRect(500, 500, 588, 588, 1,
runner->Outputs().Tag("CROP_RECT").packets);
runner->Outputs().Tag(kCropRectTag).packets);
CheckCropRect(500 * 0.5, 500 * 0.5, 288 * 0.5, 288 * 0.5, 2,
runner->Outputs().Tag("CROP_RECT").packets);
runner->Outputs().Tag(kCropRectTag).packets);
}
TEST(ContentZoomingCalculatorTest, ResolutionChangeZoomingWithCache) {
@@ -713,18 +762,18 @@ TEST(ContentZoomingCalculatorTest, ResolutionChangeZoomingWithCache) {
config.add_input_side_packet("STATE_CACHE:state_cache");
{
auto runner = ::absl::make_unique<CalculatorRunner>(config);
runner->MutableSidePackets()->Tag("STATE_CACHE") = MakePacket<
runner->MutableSidePackets()->Tag(kStateCacheTag) = MakePacket<
mediapipe::autoflip::ContentZoomingCalculatorStateCacheType*>(&cache);
AddDetectionFrameSize(cv::Rect_<float>(.1, .1, .8, .8), 0, 1000, 1000,
runner.get());
MP_ASSERT_OK(runner->Run());
CheckCropRect(500, 500, 888, 888, 0,
runner->Outputs().Tag("CROP_RECT").packets);
runner->Outputs().Tag(kCropRectTag).packets);
}
// The second runner should just resume based on state from the first runner.
{
auto runner = ::absl::make_unique<CalculatorRunner>(config);
runner->MutableSidePackets()->Tag("STATE_CACHE") = MakePacket<
runner->MutableSidePackets()->Tag(kStateCacheTag) = MakePacket<
mediapipe::autoflip::ContentZoomingCalculatorStateCacheType*>(&cache);
AddDetectionFrameSize(cv::Rect_<float>(.4, .4, .2, .2), 1000000, 1000, 1000,
runner.get());
@@ -732,9 +781,9 @@ TEST(ContentZoomingCalculatorTest, ResolutionChangeZoomingWithCache) {
runner.get());
MP_ASSERT_OK(runner->Run());
CheckCropRect(500, 500, 588, 588, 0,
runner->Outputs().Tag("CROP_RECT").packets);
runner->Outputs().Tag(kCropRectTag).packets);
CheckCropRect(500 * 0.5, 500 * 0.5, 288 * 0.5, 288 * 0.5, 1,
runner->Outputs().Tag("CROP_RECT").packets);
runner->Outputs().Tag(kCropRectTag).packets);
}
}
@@ -749,7 +798,7 @@ TEST(ContentZoomingCalculatorTest, MaxZoomValue) {
MP_ASSERT_OK(runner->Run());
// 55/60 * 1000 = 916
CheckCropRect(500, 500, 916, 916, 0,
runner->Outputs().Tag("CROP_RECT").packets);
runner->Outputs().Tag(kCropRectTag).packets);
}
TEST(ContentZoomingCalculatorTest, MaxZoomValueOverride) {
@@ -772,11 +821,11 @@ TEST(ContentZoomingCalculatorTest, MaxZoomValueOverride) {
// Max. 133% zoomed in means min. (100/133) ~ 75% of height left: ~360
// Max. 166% zoomed in means min. (100/166) ~ 60% of height left: ~430
CheckCropRect(320, 240, 480, 360, 0,
runner->Outputs().Tag("CROP_RECT").packets);
runner->Outputs().Tag(kCropRectTag).packets);
CheckCropRect(640, 360, 769, 433, 2,
runner->Outputs().Tag("CROP_RECT").packets);
runner->Outputs().Tag(kCropRectTag).packets);
CheckCropRect(320, 240, 480, 360, 3,
runner->Outputs().Tag("CROP_RECT").packets);
runner->Outputs().Tag(kCropRectTag).packets);
}
TEST(ContentZoomingCalculatorTest, MaxZoomOutValue) {
@@ -795,9 +844,9 @@ TEST(ContentZoomingCalculatorTest, MaxZoomOutValue) {
MP_ASSERT_OK(runner->Run());
// 55/60 * 1000 = 916
CheckCropRect(500, 500, 950, 950, 0,
runner->Outputs().Tag("CROP_RECT").packets);
runner->Outputs().Tag(kCropRectTag).packets);
CheckCropRect(500, 500, 1000, 1000, 2,
runner->Outputs().Tag("CROP_RECT").packets);
runner->Outputs().Tag(kCropRectTag).packets);
}
TEST(ContentZoomingCalculatorTest, StartZoomedOut) {
@@ -816,13 +865,13 @@ TEST(ContentZoomingCalculatorTest, StartZoomedOut) {
runner.get());
MP_ASSERT_OK(runner->Run());
CheckCropRect(500, 500, 1000, 1000, 0,
runner->Outputs().Tag("CROP_RECT").packets);
runner->Outputs().Tag(kCropRectTag).packets);
CheckCropRect(500, 500, 880, 880, 1,
runner->Outputs().Tag("CROP_RECT").packets);
runner->Outputs().Tag(kCropRectTag).packets);
CheckCropRect(500, 500, 760, 760, 2,
runner->Outputs().Tag("CROP_RECT").packets);
runner->Outputs().Tag(kCropRectTag).packets);
CheckCropRect(500, 500, 655, 655, 3,
runner->Outputs().Tag("CROP_RECT").packets);
runner->Outputs().Tag(kCropRectTag).packets);
}
TEST(ContentZoomingCalculatorTest, AnimateToFirstRect) {
@@ -844,15 +893,15 @@ TEST(ContentZoomingCalculatorTest, AnimateToFirstRect) {
runner.get());
MP_ASSERT_OK(runner->Run());
CheckCropRect(500, 500, 1000, 1000, 0,
runner->Outputs().Tag("CROP_RECT").packets);
runner->Outputs().Tag(kCropRectTag).packets);
CheckCropRect(500, 500, 1000, 1000, 1,
runner->Outputs().Tag("CROP_RECT").packets);
runner->Outputs().Tag(kCropRectTag).packets);
CheckCropRect(500, 500, 470, 470, 2,
runner->Outputs().Tag("CROP_RECT").packets);
runner->Outputs().Tag(kCropRectTag).packets);
CheckCropRect(500, 500, 222, 222, 3,
runner->Outputs().Tag("CROP_RECT").packets);
runner->Outputs().Tag(kCropRectTag).packets);
CheckCropRect(500, 500, 222, 222, 4,
runner->Outputs().Tag("CROP_RECT").packets);
runner->Outputs().Tag(kCropRectTag).packets);
}
TEST(ContentZoomingCalculatorTest, CanControlAnimation) {
@@ -879,15 +928,15 @@ TEST(ContentZoomingCalculatorTest, CanControlAnimation) {
runner.get(), {.animated_zoom = false});
MP_ASSERT_OK(runner->Run());
CheckCropRect(500, 500, 1000, 1000, 0,
runner->Outputs().Tag("CROP_RECT").packets);
runner->Outputs().Tag(kCropRectTag).packets);
CheckCropRect(500, 500, 1000, 1000, 1,
runner->Outputs().Tag("CROP_RECT").packets);
runner->Outputs().Tag(kCropRectTag).packets);
CheckCropRect(500, 500, 470, 470, 2,
runner->Outputs().Tag("CROP_RECT").packets);
runner->Outputs().Tag(kCropRectTag).packets);
CheckCropRect(500, 500, 222, 222, 3,
runner->Outputs().Tag("CROP_RECT").packets);
runner->Outputs().Tag(kCropRectTag).packets);
CheckCropRect(500, 500, 222, 222, 4,
runner->Outputs().Tag("CROP_RECT").packets);
runner->Outputs().Tag(kCropRectTag).packets);
}
TEST(ContentZoomingCalculatorTest, DoesNotAnimateIfDisabledViaInput) {
@@ -907,11 +956,11 @@ TEST(ContentZoomingCalculatorTest, DoesNotAnimateIfDisabledViaInput) {
runner.get(), {.animated_zoom = false});
MP_ASSERT_OK(runner->Run());
CheckCropRect(500, 500, 1000, 1000, 0,
runner->Outputs().Tag("CROP_RECT").packets);
runner->Outputs().Tag(kCropRectTag).packets);
CheckCropRect(500, 500, 880, 880, 1,
runner->Outputs().Tag("CROP_RECT").packets);
runner->Outputs().Tag(kCropRectTag).packets);
CheckCropRect(500, 500, 760, 760, 2,
runner->Outputs().Tag("CROP_RECT").packets);
runner->Outputs().Tag(kCropRectTag).packets);
}
TEST(ContentZoomingCalculatorTest, ProvidesZeroSizeFirstRectWithoutDetections) {
@@ -920,13 +969,13 @@ TEST(ContentZoomingCalculatorTest, ProvidesZeroSizeFirstRectWithoutDetections) {
auto input_size = ::absl::make_unique<std::pair<int, int>>(1000, 1000);
runner->MutableInputs()
->Tag("VIDEO_SIZE")
->Tag(kVideoSizeTag)
.packets.push_back(Adopt(input_size.release()).At(Timestamp(0)));
MP_ASSERT_OK(runner->Run());
const std::vector<Packet>& output_packets =
runner->Outputs().Tag("FIRST_CROP_RECT").packets;
runner->Outputs().Tag(kFirstCropRectTag).packets;
ASSERT_EQ(output_packets.size(), 1);
const auto& rect = output_packets[0].Get<mediapipe::NormalizedRect>();
EXPECT_EQ(rect.x_center(), 0);
@@ -951,7 +1000,7 @@ TEST(ContentZoomingCalculatorTest, ProvidesConstantFirstRect) {
runner.get());
MP_ASSERT_OK(runner->Run());
const std::vector<Packet>& output_packets =
runner->Outputs().Tag("FIRST_CROP_RECT").packets;
runner->Outputs().Tag(kFirstCropRectTag).packets;
ASSERT_EQ(output_packets.size(), 4);
const auto& first_rect = output_packets[0].Get<mediapipe::NormalizedRect>();
EXPECT_NEAR(first_rect.x_center(), 0.5, 0.05);
mediapipe/examples/desktop/autoflip/calculators/face_box_adjuster_calculator.proto
+1 -1
View File
@@ -64,7 +64,7 @@ message FaceBoxAdjusterCalculatorOptions {
// Max value of head motion (max of current or history) to be considered still
// stable.
optional float head_motion_threshold = 14 [default = 10.0];
optional float head_motion_threshold = 14 [default = 360.0];
// The max amount of time to use an old eye distance when the face look angle
// is unstable.
mediapipe/examples/desktop/autoflip/calculators/face_to_region_calculator.cc
+18 -12
View File
@@ -32,6 +32,10 @@
namespace mediapipe {
namespace autoflip {
constexpr char kRegionsTag[] = "REGIONS";
constexpr char kFacesTag[] = "FACES";
constexpr char kVideoTag[] = "VIDEO";
// This calculator converts detected faces to SalientRegion protos that can be
// used for downstream processing. Each SalientRegion is scored using image
// cues. Scoring can be controlled through
@@ -80,17 +84,17 @@ FaceToRegionCalculator::FaceToRegionCalculator() {}
absl::Status FaceToRegionCalculator::GetContract(
mediapipe::CalculatorContract* cc) {
if (cc->Inputs().HasTag("VIDEO")) {
cc->Inputs().Tag("VIDEO").Set<ImageFrame>();
if (cc->Inputs().HasTag(kVideoTag)) {
cc->Inputs().Tag(kVideoTag).Set<ImageFrame>();
}
cc->Inputs().Tag("FACES").Set<std::vector<mediapipe::Detection>>();
cc->Outputs().Tag("REGIONS").Set<DetectionSet>();
cc->Inputs().Tag(kFacesTag).Set<std::vector<mediapipe::Detection>>();
cc->Outputs().Tag(kRegionsTag).Set<DetectionSet>();
return absl::OkStatus();
}
absl::Status FaceToRegionCalculator::Open(mediapipe::CalculatorContext* cc) {
options_ = cc->Options<FaceToRegionCalculatorOptions>();
if (!cc->Inputs().HasTag("VIDEO")) {
if (!cc->Inputs().HasTag(kVideoTag)) {
RET_CHECK(!options_.use_visual_scorer())
<< "VIDEO input must be provided when using visual_scorer.";
RET_CHECK(!options_.export_individual_face_landmarks())
@@ -146,24 +150,24 @@ void FaceToRegionCalculator::ExtendSalientRegionWithPoint(
}
absl::Status FaceToRegionCalculator::Process(mediapipe::CalculatorContext* cc) {
if (cc->Inputs().HasTag("VIDEO") &&
cc->Inputs().Tag("VIDEO").Value().IsEmpty()) {
if (cc->Inputs().HasTag(kVideoTag) &&
cc->Inputs().Tag(kVideoTag).Value().IsEmpty()) {
return mediapipe::UnknownErrorBuilder(MEDIAPIPE_LOC)
<< "No VIDEO input at time " << cc->InputTimestamp().Seconds();
}
cv::Mat frame;
if (cc->Inputs().HasTag("VIDEO")) {
if (cc->Inputs().HasTag(kVideoTag)) {
frame = mediapipe::formats::MatView(
&cc->Inputs().Tag("VIDEO").Get<ImageFrame>());
&cc->Inputs().Tag(kVideoTag).Get<ImageFrame>());
frame_width_ = frame.cols;
frame_height_ = frame.rows;
}
auto region_set = ::absl::make_unique<DetectionSet>();
if (!cc->Inputs().Tag("FACES").Value().IsEmpty()) {
if (!cc->Inputs().Tag(kFacesTag).Value().IsEmpty()) {
const auto& input_faces =
cc->Inputs().Tag("FACES").Get<std::vector<mediapipe::Detection>>();
cc->Inputs().Tag(kFacesTag).Get<std::vector<mediapipe::Detection>>();
for (const auto& input_face : input_faces) {
RET_CHECK(input_face.location_data().format() ==
@@ -276,7 +280,9 @@ absl::Status FaceToRegionCalculator::Process(mediapipe::CalculatorContext* cc) {
}
}
}
cc->Outputs().Tag("REGIONS").Add(region_set.release(), cc->InputTimestamp());
cc->Outputs()
.Tag(kRegionsTag)
.Add(region_set.release(), cc->InputTimestamp());
return absl::OkStatus();
}
mediapipe/examples/desktop/autoflip/calculators/face_to_region_calculator_test.cc
+11 -7
View File
@@ -33,6 +33,10 @@ namespace mediapipe {
namespace autoflip {
namespace {
constexpr char kRegionsTag[] = "REGIONS";
constexpr char kFacesTag[] = "FACES";
constexpr char kVideoTag[] = "VIDEO";
const char kConfig[] = R"(
calculator: "FaceToRegionCalculator"
input_stream: "VIDEO:frames"
@@ -100,7 +104,7 @@ void SetInputs(const std::vector<std::string>& faces, const bool include_video,
if (include_video) {
auto input_frame =
::absl::make_unique<ImageFrame>(ImageFormat::SRGB, 800, 600);
runner->MutableInputs()->Tag("VIDEO").packets.push_back(
runner->MutableInputs()->Tag(kVideoTag).packets.push_back(
Adopt(input_frame.release()).At(Timestamp::PostStream()));
}
// Setup two faces as input.
@@ -109,7 +113,7 @@ void SetInputs(const std::vector<std::string>& faces, const bool include_video,
for (const auto& face : faces) {
input_faces->push_back(ParseTextProtoOrDie<Detection>(face));
}
runner->MutableInputs()->Tag("FACES").packets.push_back(
runner->MutableInputs()->Tag(kFacesTag).packets.push_back(
Adopt(input_faces.release()).At(Timestamp::PostStream()));
}
@@ -144,7 +148,7 @@ TEST(FaceToRegionCalculatorTest, FaceFullTypeSize) {
// Check the output regions.
const std::vector<Packet>& output_packets =
runner->Outputs().Tag("REGIONS").packets;
runner->Outputs().Tag(kRegionsTag).packets;
ASSERT_EQ(1, output_packets.size());
const auto& regions = output_packets[0].Get<DetectionSet>();
@@ -177,7 +181,7 @@ TEST(FaceToRegionCalculatorTest, FaceLandmarksTypeSize) {
// Check the output regions.
const std::vector<Packet>& output_packets =
runner->Outputs().Tag("REGIONS").packets;
runner->Outputs().Tag(kRegionsTag).packets;
ASSERT_EQ(1, output_packets.size());
const auto& regions = output_packets[0].Get<DetectionSet>();
@@ -208,7 +212,7 @@ TEST(FaceToRegionCalculatorTest, FaceLandmarksBox) {
// Check the output regions.
const std::vector<Packet>& output_packets =
runner->Outputs().Tag("REGIONS").packets;
runner->Outputs().Tag(kRegionsTag).packets;
ASSERT_EQ(1, output_packets.size());
const auto& regions = output_packets[0].Get<DetectionSet>();
@@ -243,7 +247,7 @@ TEST(FaceToRegionCalculatorTest, FaceScore) {
// Check the output regions.
const std::vector<Packet>& output_packets =
runner->Outputs().Tag("REGIONS").packets;
runner->Outputs().Tag(kRegionsTag).packets;
ASSERT_EQ(1, output_packets.size());
const auto& regions = output_packets[0].Get<DetectionSet>();
ASSERT_EQ(1, regions.detections().size());
@@ -292,7 +296,7 @@ TEST(FaceToRegionCalculatorTest, FaceNoVideoPass) {
// Check the output regions.
const std::vector<Packet>& output_packets =
runner->Outputs().Tag("REGIONS").packets;
runner->Outputs().Tag(kRegionsTag).packets;
ASSERT_EQ(1, output_packets.size());
const auto& regions = output_packets[0].Get<DetectionSet>();
mediapipe/examples/desktop/autoflip/calculators/localization_to_region_calculator.cc
+7 -4
View File
@@ -52,6 +52,9 @@ LocalizationToRegionCalculator::LocalizationToRegionCalculator() {}
namespace {
constexpr char kRegionsTag[] = "REGIONS";
constexpr char kDetectionsTag[] = "DETECTIONS";
// Converts an object detection to a autoflip SignalType. Returns true if the
// std::string label has a autoflip label.
bool MatchType(const std::string& label, SignalType* type) {
@@ -86,8 +89,8 @@ void FillSalientRegion(const mediapipe::Detection& detection,
absl::Status LocalizationToRegionCalculator::GetContract(
mediapipe::CalculatorContract* cc) {
cc->Inputs().Tag("DETECTIONS").Set<std::vector<mediapipe::Detection>>();
cc->Outputs().Tag("REGIONS").Set<DetectionSet>();
cc->Inputs().Tag(kDetectionsTag).Set<std::vector<mediapipe::Detection>>();
cc->Outputs().Tag(kRegionsTag).Set<DetectionSet>();
return absl::OkStatus();
}
@@ -101,7 +104,7 @@ absl::Status LocalizationToRegionCalculator::Open(
absl::Status LocalizationToRegionCalculator::Process(
mediapipe::CalculatorContext* cc) {
const auto& annotations =
cc->Inputs().Tag("DETECTIONS").Get<std::vector<mediapipe::Detection>>();
cc->Inputs().Tag(kDetectionsTag).Get<std::vector<mediapipe::Detection>>();
auto regions = ::absl::make_unique<DetectionSet>();
for (const auto& detection : annotations) {
RET_CHECK_EQ(detection.label().size(), 1)
@@ -118,7 +121,7 @@ absl::Status LocalizationToRegionCalculator::Process(
}
}
cc->Outputs().Tag("REGIONS").Add(regions.release(), cc->InputTimestamp());
cc->Outputs().Tag(kRegionsTag).Add(regions.release(), cc->InputTimestamp());
return absl::OkStatus();
}
mediapipe/examples/desktop/autoflip/calculators/localization_to_region_calculator_test.cc
+7 -4
View File
@@ -31,6 +31,9 @@ namespace mediapipe {
namespace autoflip {
namespace {
constexpr char kRegionsTag[] = "REGIONS";
constexpr char kDetectionsTag[] = "DETECTIONS";
const char kConfig[] = R"(
calculator: "LocalizationToRegionCalculator"
input_stream: "DETECTIONS:detections"
@@ -81,7 +84,7 @@ void SetInputs(CalculatorRunner* runner,
inputs->push_back(ParseTextProtoOrDie<Detection>(detection));
}
runner->MutableInputs()
->Tag("DETECTIONS")
->Tag(kDetectionsTag)
.packets.push_back(Adopt(inputs.release()).At(Timestamp::PostStream()));
}
@@ -109,7 +112,7 @@ TEST(LocalizationToRegionCalculatorTest, StandardTypes) {
// Check the output regions.
const std::vector<Packet>& output_packets =
runner->Outputs().Tag("REGIONS").packets;
runner->Outputs().Tag(kRegionsTag).packets;
ASSERT_EQ(1, output_packets.size());
const auto& regions = output_packets[0].Get<DetectionSet>();
ASSERT_EQ(2, regions.detections().size());
@@ -137,7 +140,7 @@ TEST(LocalizationToRegionCalculatorTest, AllTypes) {
// Check the output regions.
const std::vector<Packet>& output_packets =
runner->Outputs().Tag("REGIONS").packets;
runner->Outputs().Tag(kRegionsTag).packets;
ASSERT_EQ(1, output_packets.size());
const auto& regions = output_packets[0].Get<DetectionSet>();
ASSERT_EQ(3, regions.detections().size());
@@ -153,7 +156,7 @@ TEST(LocalizationToRegionCalculatorTest, BothTypes) {
// Check the output regions.
const std::vector<Packet>& output_packets =
runner->Outputs().Tag("REGIONS").packets;
runner->Outputs().Tag(kRegionsTag).packets;
ASSERT_EQ(1, output_packets.size());
const auto& regions = output_packets[0].Get<DetectionSet>();
ASSERT_EQ(5, regions.detections().size());
mediapipe/examples/desktop/autoflip/calculators/scene_cropping_calculator_test.cc
+53 -36
View File
@@ -34,6 +34,23 @@ namespace mediapipe {
namespace autoflip {
namespace {
constexpr char kFramingDetectionsVizFramesTag[] =
"FRAMING_DETECTIONS_VIZ_FRAMES";
constexpr char kExternalRenderingFullVidTag[] = "EXTERNAL_RENDERING_FULL_VID";
constexpr char kExternalRenderingPerFrameTag[] = "EXTERNAL_RENDERING_PER_FRAME";
constexpr char kCroppingSummaryTag[] = "CROPPING_SUMMARY";
constexpr char kSalientPointFrameVizFramesTag[] =
"SALIENT_POINT_FRAME_VIZ_FRAMES";
constexpr char kKeyFrameCropRegionVizFramesTag[] =
"KEY_FRAME_CROP_REGION_VIZ_FRAMES";
constexpr char kCroppedFramesTag[] = "CROPPED_FRAMES";
constexpr char kShotBoundariesTag[] = "SHOT_BOUNDARIES";
constexpr char kStaticFeaturesTag[] = "STATIC_FEATURES";
constexpr char kVideoSizeTag[] = "VIDEO_SIZE";
constexpr char kVideoFramesTag[] = "VIDEO_FRAMES";
constexpr char kDetectionFeaturesTag[] = "DETECTION_FEATURES";
constexpr char kKeyFramesTag[] = "KEY_FRAMES";
using ::testing::HasSubstr;
constexpr char kConfig[] = R"(
@@ -241,10 +258,10 @@ void AddKeyFrameFeatures(const int64 time_ms, const int key_frame_width,
const int key_frame_height, bool randomize,
CalculatorRunner::StreamContentsSet* inputs) {
Timestamp timestamp(time_ms);
if (inputs->HasTag("KEY_FRAMES")) {
if (inputs->HasTag(kKeyFramesTag)) {
auto key_frame = MakeImageFrameFromColor(GetRandomColor(), key_frame_width,
key_frame_height);
inputs->Tag("KEY_FRAMES")
inputs->Tag(kKeyFramesTag)
.packets.push_back(Adopt(key_frame.release()).At(timestamp));
}
if (randomize) {
@@ -252,11 +269,11 @@ void AddKeyFrameFeatures(const int64 time_ms, const int key_frame_width,
kMinNumDetections, kMaxNumDetections)(GetGen());
auto detections =
MakeDetections(num_detections, key_frame_width, key_frame_height);
inputs->Tag("DETECTION_FEATURES")
inputs->Tag(kDetectionFeaturesTag)
.packets.push_back(Adopt(detections.release()).At(timestamp));
} else {
auto detections = MakeCenterDetection(key_frame_width, key_frame_height);
inputs->Tag("DETECTION_FEATURES")
inputs->Tag(kDetectionFeaturesTag)
.packets.push_back(Adopt(detections.release()).At(timestamp));
}
}
@@ -272,19 +289,19 @@ void AddScene(const int start_frame_index, const int num_scene_frames,
int64 time_ms = start_frame_index * kTimestampDiff;
for (int i = 0; i < num_scene_frames; ++i) {
Timestamp timestamp(time_ms);
if (inputs->HasTag("VIDEO_FRAMES")) {
if (inputs->HasTag(kVideoFramesTag)) {
auto frame =
MakeImageFrameFromColor(GetRandomColor(), frame_width, frame_height);
inputs->Tag("VIDEO_FRAMES")
inputs->Tag(kVideoFramesTag)
.packets.push_back(Adopt(frame.release()).At(timestamp));
} else {
auto input_size =
::absl::make_unique<std::pair<int, int>>(frame_width, frame_height);
inputs->Tag("VIDEO_SIZE")
inputs->Tag(kVideoSizeTag)
.packets.push_back(Adopt(input_size.release()).At(timestamp));
}
auto static_features = absl::make_unique<StaticFeatures>();
inputs->Tag("STATIC_FEATURES")
inputs->Tag(kStaticFeaturesTag)
.packets.push_back(Adopt(static_features.release()).At(timestamp));
if (DownSampleRate == 1) {
AddKeyFrameFeatures(time_ms, key_frame_width, key_frame_height, false,
@@ -294,7 +311,7 @@ void AddScene(const int start_frame_index, const int num_scene_frames,
inputs);
}
if (i == num_scene_frames - 1) { // adds shot boundary
inputs->Tag("SHOT_BOUNDARIES")
inputs->Tag(kShotBoundariesTag)
.packets.push_back(Adopt(new bool(true)).At(Timestamp(time_ms)));
}
time_ms += kTimestampDiff;
@@ -306,8 +323,8 @@ void AddScene(const int start_frame_index, const int num_scene_frames,
void CheckCroppedFrames(const CalculatorRunner& runner, const int num_frames,
const int target_width, const int target_height) {
const auto& outputs = runner.Outputs();
EXPECT_TRUE(outputs.HasTag("CROPPED_FRAMES"));
const auto& cropped_frames_outputs = outputs.Tag("CROPPED_FRAMES").packets;
EXPECT_TRUE(outputs.HasTag(kCroppedFramesTag));
const auto& cropped_frames_outputs = outputs.Tag(kCroppedFramesTag).packets;
EXPECT_EQ(cropped_frames_outputs.size(), num_frames);
for (int i = 0; i < num_frames; ++i) {
const auto& cropped_frame = cropped_frames_outputs[i].Get<ImageFrame>();
@@ -392,23 +409,23 @@ TEST(SceneCroppingCalculatorTest, OutputsDebugStreams) {
MP_EXPECT_OK(runner->Run());
const auto& outputs = runner->Outputs();
EXPECT_TRUE(outputs.HasTag("KEY_FRAME_CROP_REGION_VIZ_FRAMES"));
EXPECT_TRUE(outputs.HasTag("SALIENT_POINT_FRAME_VIZ_FRAMES"));
EXPECT_TRUE(outputs.HasTag("CROPPING_SUMMARY"));
EXPECT_TRUE(outputs.HasTag("EXTERNAL_RENDERING_PER_FRAME"));
EXPECT_TRUE(outputs.HasTag("EXTERNAL_RENDERING_FULL_VID"));
EXPECT_TRUE(outputs.HasTag("FRAMING_DETECTIONS_VIZ_FRAMES"));
EXPECT_TRUE(outputs.HasTag(kKeyFrameCropRegionVizFramesTag));
EXPECT_TRUE(outputs.HasTag(kSalientPointFrameVizFramesTag));
EXPECT_TRUE(outputs.HasTag(kCroppingSummaryTag));
EXPECT_TRUE(outputs.HasTag(kExternalRenderingPerFrameTag));
EXPECT_TRUE(outputs.HasTag(kExternalRenderingFullVidTag));
EXPECT_TRUE(outputs.HasTag(kFramingDetectionsVizFramesTag));
const auto& crop_region_viz_frames_outputs =
outputs.Tag("KEY_FRAME_CROP_REGION_VIZ_FRAMES").packets;
outputs.Tag(kKeyFrameCropRegionVizFramesTag).packets;
const auto& salient_point_viz_frames_outputs =
outputs.Tag("SALIENT_POINT_FRAME_VIZ_FRAMES").packets;
const auto& summary_output = outputs.Tag("CROPPING_SUMMARY").packets;
outputs.Tag(kSalientPointFrameVizFramesTag).packets;
const auto& summary_output = outputs.Tag(kCroppingSummaryTag).packets;
const auto& ext_render_per_frame =
outputs.Tag("EXTERNAL_RENDERING_PER_FRAME").packets;
outputs.Tag(kExternalRenderingPerFrameTag).packets;
const auto& ext_render_full_vid =
outputs.Tag("EXTERNAL_RENDERING_FULL_VID").packets;
outputs.Tag(kExternalRenderingFullVidTag).packets;
const auto& framing_viz_frames_output =
outputs.Tag("FRAMING_DETECTIONS_VIZ_FRAMES").packets;
outputs.Tag(kFramingDetectionsVizFramesTag).packets;
EXPECT_EQ(crop_region_viz_frames_outputs.size(), num_frames);
EXPECT_EQ(salient_point_viz_frames_outputs.size(), num_frames);
EXPECT_EQ(framing_viz_frames_output.size(), num_frames);
@@ -597,7 +614,7 @@ TEST(SceneCroppingCalculatorTest, ProducesEvenFrameSize) {
kKeyFrameHeight, kDownSampleRate, runner->MutableInputs());
MP_EXPECT_OK(runner->Run());
const auto& output_frame = runner->Outputs()
.Tag("CROPPED_FRAMES")
.Tag(kCroppedFramesTag)
.packets[0]
.Get<ImageFrame>();
EXPECT_EQ(output_frame.Width() % 2, 0);
@@ -646,7 +663,7 @@ TEST(SceneCroppingCalculatorTest, PadsWithSolidColorFromStaticFeatures) {
Timestamp timestamp(time_ms);
auto frame =
MakeImageFrameFromColor(GetRandomColor(), input_width, input_height);
inputs->Tag("VIDEO_FRAMES")
inputs->Tag(kVideoFramesTag)
.packets.push_back(Adopt(frame.release()).At(timestamp));
if (i % static_features_downsample_rate == 0) {
auto static_features = absl::make_unique<StaticFeatures>();
@@ -657,7 +674,7 @@ TEST(SceneCroppingCalculatorTest, PadsWithSolidColorFromStaticFeatures) {
color->set_g(green);
color->set_b(red);
}
inputs->Tag("STATIC_FEATURES")
inputs->Tag(kStaticFeaturesTag)
.packets.push_back(Adopt(static_features.release()).At(timestamp));
num_static_features++;
}
@@ -672,7 +689,7 @@ TEST(SceneCroppingCalculatorTest, PadsWithSolidColorFromStaticFeatures) {
location->set_y(0);
location->set_width(80);
location->set_height(input_height);
inputs->Tag("DETECTION_FEATURES")
inputs->Tag(kDetectionFeaturesTag)
.packets.push_back(Adopt(detections.release()).At(timestamp));
}
time_ms += kTimestampDiff;
@@ -683,7 +700,7 @@ TEST(SceneCroppingCalculatorTest, PadsWithSolidColorFromStaticFeatures) {
// Checks that the top and bottom borders indeed have the background color.
const int border_size = 37;
const auto& cropped_frames_outputs =
runner->Outputs().Tag("CROPPED_FRAMES").packets;
runner->Outputs().Tag(kCroppedFramesTag).packets;
EXPECT_EQ(cropped_frames_outputs.size(), kSceneSize);
for (int i = 0; i < kSceneSize; ++i) {
const auto& cropped_frame = cropped_frames_outputs[i].Get<ImageFrame>();
@@ -727,7 +744,7 @@ TEST(SceneCroppingCalculatorTest, RemovesStaticBorders) {
auto mat = formats::MatView(frame.get());
mat(top_border_rect) = border_color;
mat(bottom_border_rect) = border_color;
inputs->Tag("VIDEO_FRAMES")
inputs->Tag(kVideoFramesTag)
.packets.push_back(Adopt(frame.release()).At(timestamp));
// Set borders in static features.
auto static_features = absl::make_unique<StaticFeatures>();
@@ -737,11 +754,11 @@ TEST(SceneCroppingCalculatorTest, RemovesStaticBorders) {
auto* bottom_part = static_features->add_border();
bottom_part->set_relative_position(Border::BOTTOM);
bottom_part->mutable_border_position()->set_height(bottom_border_size);
inputs->Tag("STATIC_FEATURES")
inputs->Tag(kStaticFeaturesTag)
.packets.push_back(Adopt(static_features.release()).At(timestamp));
// Add empty detections to ensure no padding is used.
auto detections = absl::make_unique<DetectionSet>();
inputs->Tag("DETECTION_FEATURES")
inputs->Tag(kDetectionFeaturesTag)
.packets.push_back(Adopt(detections.release()).At(timestamp));
MP_EXPECT_OK(runner->Run());
@@ -749,7 +766,7 @@ TEST(SceneCroppingCalculatorTest, RemovesStaticBorders) {
// Checks that the top and bottom borders are removed. Each frame should have
// solid color equal to frame color.
const auto& cropped_frames_outputs =
runner->Outputs().Tag("CROPPED_FRAMES").packets;
runner->Outputs().Tag(kCroppedFramesTag).packets;
EXPECT_EQ(cropped_frames_outputs.size(), 1);
const auto& cropped_frame = cropped_frames_outputs[0].Get<ImageFrame>();
const auto cropped_mat = formats::MatView(&cropped_frame);
@@ -775,7 +792,7 @@ TEST(SceneCroppingCalculatorTest, OutputsCropMessagePolyPath) {
MP_EXPECT_OK(runner->Run());
const auto& outputs = runner->Outputs();
const auto& ext_render_per_frame =
outputs.Tag("EXTERNAL_RENDERING_PER_FRAME").packets;
outputs.Tag(kExternalRenderingPerFrameTag).packets;
EXPECT_EQ(ext_render_per_frame.size(), num_frames);
for (int i = 0; i < num_frames - 1; ++i) {
@@ -813,7 +830,7 @@ TEST(SceneCroppingCalculatorTest, OutputsCropMessageKinematicPath) {
MP_EXPECT_OK(runner->Run());
const auto& outputs = runner->Outputs();
const auto& ext_render_per_frame =
outputs.Tag("EXTERNAL_RENDERING_PER_FRAME").packets;
outputs.Tag(kExternalRenderingPerFrameTag).packets;
EXPECT_EQ(ext_render_per_frame.size(), num_frames);
for (int i = 0; i < num_frames - 1; ++i) {
@@ -846,7 +863,7 @@ TEST(SceneCroppingCalculatorTest, OutputsCropMessagePolyPathNoVideo) {
MP_EXPECT_OK(runner->Run());
const auto& outputs = runner->Outputs();
const auto& ext_render_per_frame =
outputs.Tag("EXTERNAL_RENDERING_PER_FRAME").packets;
outputs.Tag(kExternalRenderingPerFrameTag).packets;
EXPECT_EQ(ext_render_per_frame.size(), num_frames);
for (int i = 0; i < num_frames - 1; ++i) {
@@ -886,7 +903,7 @@ TEST(SceneCroppingCalculatorTest, OutputsCropMessageKinematicPathNoVideo) {
MP_EXPECT_OK(runner->Run());
const auto& outputs = runner->Outputs();
const auto& ext_render_per_frame =
outputs.Tag("EXTERNAL_RENDERING_PER_FRAME").packets;
outputs.Tag(kExternalRenderingPerFrameTag).packets;
EXPECT_EQ(ext_render_per_frame.size(), num_frames);
for (int i = 0; i < num_frames - 1; ++i) {
mediapipe/examples/desktop/autoflip/calculators/shot_boundary_calculator_test.cc
+9 -6
View File
@@ -43,6 +43,9 @@ namespace mediapipe {
namespace autoflip {
namespace {
constexpr char kIsShotChangeTag[] = "IS_SHOT_CHANGE";
constexpr char kVideoTag[] = "VIDEO";
const char kConfig[] = R"(
calculator: "ShotBoundaryCalculator"
input_stream: "VIDEO:camera_frames"
@@ -70,7 +73,7 @@ void AddFrames(const int number_of_frames, const std::set<int>& skip_frames,
if (skip_frames.count(i) < 1) {
sub_image.copyTo(frame_area);
}
runner->MutableInputs()->Tag("VIDEO").packets.push_back(
runner->MutableInputs()->Tag(kVideoTag).packets.push_back(
Adopt(input_frame.release()).At(Timestamp(i * 1000000)));
}
}
@@ -97,7 +100,7 @@ TEST(ShotBoundaryCalculatorTest, NoShotChange) {
AddFrames(10, {}, runner.get());
MP_ASSERT_OK(runner->Run());
CheckOutput(10, {}, runner->Outputs().Tag("IS_SHOT_CHANGE").packets);
CheckOutput(10, {}, runner->Outputs().Tag(kIsShotChangeTag).packets);
}
TEST(ShotBoundaryCalculatorTest, ShotChangeSingle) {
@@ -110,7 +113,7 @@ TEST(ShotBoundaryCalculatorTest, ShotChangeSingle) {
AddFrames(20, {10}, runner.get());
MP_ASSERT_OK(runner->Run());
CheckOutput(20, {10}, runner->Outputs().Tag("IS_SHOT_CHANGE").packets);
CheckOutput(20, {10}, runner->Outputs().Tag(kIsShotChangeTag).packets);
}
TEST(ShotBoundaryCalculatorTest, ShotChangeDouble) {
@@ -123,7 +126,7 @@ TEST(ShotBoundaryCalculatorTest, ShotChangeDouble) {
AddFrames(20, {14, 17}, runner.get());
MP_ASSERT_OK(runner->Run());
CheckOutput(20, {14, 17}, runner->Outputs().Tag("IS_SHOT_CHANGE").packets);
CheckOutput(20, {14, 17}, runner->Outputs().Tag(kIsShotChangeTag).packets);
}
TEST(ShotBoundaryCalculatorTest, ShotChangeFiltered) {
@@ -140,7 +143,7 @@ TEST(ShotBoundaryCalculatorTest, ShotChangeFiltered) {
AddFrames(24, {16, 19}, runner.get());
MP_ASSERT_OK(runner->Run());
CheckOutput(24, {16}, runner->Outputs().Tag("IS_SHOT_CHANGE").packets);
CheckOutput(24, {16}, runner->Outputs().Tag(kIsShotChangeTag).packets);
}
TEST(ShotBoundaryCalculatorTest, ShotChangeSingleOnOnChange) {
@@ -153,7 +156,7 @@ TEST(ShotBoundaryCalculatorTest, ShotChangeSingleOnOnChange) {
AddFrames(20, {15}, runner.get());
MP_ASSERT_OK(runner->Run());
auto output_packets = runner->Outputs().Tag("IS_SHOT_CHANGE").packets;
auto output_packets = runner->Outputs().Tag(kIsShotChangeTag).packets;
ASSERT_EQ(output_packets.size(), 1);
ASSERT_EQ(output_packets[0].Get<bool>(), true);
ASSERT_EQ(output_packets[0].Timestamp().Value(), 15000000);
mediapipe/examples/desktop/autoflip/calculators/signal_fusing_calculator_test.cc
+6 -3
View File
@@ -32,6 +32,9 @@ namespace mediapipe {
namespace autoflip {
namespace {
constexpr char kOutputTag[] = "OUTPUT";
constexpr char kIsShotBoundaryTag[] = "IS_SHOT_BOUNDARY";
const char kConfigA[] = R"(
calculator: "SignalFusingCalculator"
input_stream: "scene_change"
@@ -160,7 +163,7 @@ TEST(SignalFusingCalculatorTest, TwoInputShotLabeledTags) {
auto input_shot = absl::make_unique<bool>(false);
runner->MutableInputs()
->Tag("IS_SHOT_BOUNDARY")
->Tag(kIsShotBoundaryTag)
.packets.push_back(Adopt(input_shot.release()).At(Timestamp(0)));
auto input_face =
@@ -200,7 +203,7 @@ TEST(SignalFusingCalculatorTest, TwoInputShotLabeledTags) {
MP_ASSERT_OK(runner->Run());
const std::vector<Packet>& output_packets =
runner->Outputs().Tag("OUTPUT").packets;
runner->Outputs().Tag(kOutputTag).packets;
const auto& detection_set = output_packets[0].Get<DetectionSet>();
ASSERT_EQ(detection_set.detections().size(), 4);
@@ -251,7 +254,7 @@ TEST(SignalFusingCalculatorTest, TwoInputNoShotLabeledTags) {
MP_ASSERT_OK(runner->Run());
const std::vector<Packet>& output_packets =
runner->Outputs().Tag("OUTPUT").packets;
runner->Outputs().Tag(kOutputTag).packets;
const auto& detection_set = output_packets[0].Get<DetectionSet>();
ASSERT_EQ(detection_set.detections().size(), 4);
mediapipe/examples/desktop/autoflip/calculators/video_filtering_calculator_test.cc
+12 -9
View File
@@ -31,6 +31,9 @@ namespace mediapipe {
namespace autoflip {
namespace {
constexpr char kOutputFramesTag[] = "OUTPUT_FRAMES";
constexpr char kInputFramesTag[] = "INPUT_FRAMES";
// Default configuration of the calculator.
CalculatorGraphConfig::Node GetCalculatorNode(
const std::string& fail_if_any, const std::string& extra_options = "") {
@@ -65,10 +68,10 @@ TEST(VideoFilterCalculatorTest, UpperBoundNoPass) {
ImageFormat::SRGB, kFixedWidth,
static_cast<int>(kFixedWidth / kAspectRatio), 16);
runner->MutableInputs()
->Tag("INPUT_FRAMES")
->Tag(kInputFramesTag)
.packets.push_back(Adopt(input_frame.release()).At(Timestamp(1000)));
MP_ASSERT_OK(runner->Run());
const auto& output_packet = runner->Outputs().Tag("OUTPUT_FRAMES").packets;
const auto& output_packet = runner->Outputs().Tag(kOutputFramesTag).packets;
EXPECT_TRUE(output_packet.empty());
}
@@ -88,10 +91,10 @@ TEST(VerticalFrameRemovalCalculatorTest, UpperBoundPass) {
auto input_frame =
::absl::make_unique<ImageFrame>(ImageFormat::SRGB, kWidth, kHeight, 16);
runner->MutableInputs()
->Tag("INPUT_FRAMES")
->Tag(kInputFramesTag)
.packets.push_back(Adopt(input_frame.release()).At(Timestamp(1000)));
MP_ASSERT_OK(runner->Run());
const auto& output_packet = runner->Outputs().Tag("OUTPUT_FRAMES").packets;
const auto& output_packet = runner->Outputs().Tag(kOutputFramesTag).packets;
EXPECT_EQ(1, output_packet.size());
auto& output_frame = output_packet[0].Get<ImageFrame>();
EXPECT_EQ(kWidth, output_frame.Width());
@@ -114,10 +117,10 @@ TEST(VideoFilterCalculatorTest, LowerBoundNoPass) {
ImageFormat::SRGB, kFixedWidth,
static_cast<int>(kFixedWidth / kAspectRatio), 16);
runner->MutableInputs()
->Tag("INPUT_FRAMES")
->Tag(kInputFramesTag)
.packets.push_back(Adopt(input_frame.release()).At(Timestamp(1000)));
MP_ASSERT_OK(runner->Run());
const auto& output_packet = runner->Outputs().Tag("OUTPUT_FRAMES").packets;
const auto& output_packet = runner->Outputs().Tag(kOutputFramesTag).packets;
EXPECT_TRUE(output_packet.empty());
}
@@ -137,10 +140,10 @@ TEST(VerticalFrameRemovalCalculatorTest, LowerBoundPass) {
auto input_frame =
::absl::make_unique<ImageFrame>(ImageFormat::SRGB, kWidth, kHeight, 16);
runner->MutableInputs()
->Tag("INPUT_FRAMES")
->Tag(kInputFramesTag)
.packets.push_back(Adopt(input_frame.release()).At(Timestamp(1000)));
MP_ASSERT_OK(runner->Run());
const auto& output_packet = runner->Outputs().Tag("OUTPUT_FRAMES").packets;
const auto& output_packet = runner->Outputs().Tag(kOutputFramesTag).packets;
EXPECT_EQ(1, output_packet.size());
auto& output_frame = output_packet[0].Get<ImageFrame>();
EXPECT_EQ(kWidth, output_frame.Width());
@@ -164,7 +167,7 @@ TEST(VerticalFrameRemovalCalculatorTest, OutputError) {
ImageFormat::SRGB, kFixedWidth,
static_cast<int>(kFixedWidth / kAspectRatio), 16);
runner->MutableInputs()
->Tag("INPUT_FRAMES")
->Tag(kInputFramesTag)
.packets.push_back(Adopt(input_frame.release()).At(Timestamp(1000)));
absl::Status status = runner->Run();
EXPECT_EQ(status.code(), absl::StatusCode::kUnknown);
mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver.cc
+57 -6
View File
@@ -1,5 +1,7 @@
#include "mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver.h"
constexpr float kMinVelocity = 0.5;
namespace mediapipe {
namespace autoflip {
namespace {
@@ -75,6 +77,7 @@ absl::Status KinematicPathSolver::AddObservation(int position,
current_position_px_ = position;
}
target_position_px_ = position;
prior_position_px_ = current_position_px_;
motion_state_ = false;
mean_delta_t_ = -1;
raw_positions_at_time_.push_front(
@@ -106,6 +109,11 @@ absl::Status KinematicPathSolver::AddObservation(int position,
options_.reframe_window())
<< "Reframe window cannot exceed min_motion_to_reframe.";
}
RET_CHECK(options_.has_max_velocity() ^
(options_.has_max_velocity_scale() &&
options_.has_max_velocity_shift()))
<< "Must either set max_velocity or set both max_velocity_scale and "
"max_velocity_shift.";
return absl::OkStatus();
}
@@ -123,9 +131,29 @@ absl::Status KinematicPathSolver::AddObservation(int position,
}
int filtered_position = Median(raw_positions_at_time_);
float min_reframe = (options_.has_min_motion_to_reframe()
? options_.min_motion_to_reframe()
: options_.min_motion_to_reframe_lower()) *
pixels_per_degree_;
float max_reframe = (options_.has_min_motion_to_reframe()
? options_.min_motion_to_reframe()
: options_.min_motion_to_reframe_upper()) *
pixels_per_degree_;
filtered_position = fmax(min_location_ - min_reframe, filtered_position);
filtered_position = fmin(max_location_ + max_reframe, filtered_position);
double delta_degs =
(filtered_position - current_position_px_) / pixels_per_degree_;
double max_velocity =
options_.has_max_velocity()
? options_.max_velocity()
: fmax(abs(delta_degs * options_.max_velocity_scale()) +
options_.max_velocity_shift(),
kMinVelocity);
// If the motion is smaller than the min_motion_to_reframe and camera is
// stationary, don't use the update.
if (IsMotionTooSmall(delta_degs) && !motion_state_) {
@@ -169,10 +197,9 @@ absl::Status KinematicPathSolver::AddObservation(int position,
options_.max_update_rate());
double updated_velocity = current_velocity_deg_per_s_ * (1 - update_rate) +
observed_velocity * update_rate;
// Limited current velocity.
current_velocity_deg_per_s_ =
updated_velocity > 0 ? fmin(updated_velocity, options_.max_velocity())
: fmax(updated_velocity, -options_.max_velocity());
current_velocity_deg_per_s_ = updated_velocity > 0
? fmin(updated_velocity, max_velocity)
: fmax(updated_velocity, -max_velocity);
// Update prediction based on time input.
return UpdatePrediction(time_us);
@@ -182,6 +209,9 @@ absl::Status KinematicPathSolver::UpdatePrediction(const int64 time_us) {
RET_CHECK(current_time_ < time_us)
<< "Prediction time added before a prior observation or prediction.";
// Store prior pixel location.
prior_position_px_ = current_position_px_;
// Position update limited by min/max.
double update_position_px =
current_position_px_ +
@@ -209,7 +239,19 @@ absl::Status KinematicPathSolver::GetState(int* position) {
return absl::OkStatus();
}
absl::Status KinematicPathSolver::SetState(const int position) {
absl::Status KinematicPathSolver::GetState(float* position) {
RET_CHECK(initialized_) << "GetState called before first observation added.";
*position = current_position_px_;
return absl::OkStatus();
}
absl::Status KinematicPathSolver::GetDeltaState(float* delta_position) {
RET_CHECK(initialized_) << "GetState called before first observation added.";
*delta_position = current_position_px_ - prior_position_px_;
return absl::OkStatus();
}
absl::Status KinematicPathSolver::SetState(const float position) {
RET_CHECK(initialized_) << "SetState called before first observation added.";
current_position_px_ = position;
return absl::OkStatus();
@@ -218,7 +260,15 @@ absl::Status KinematicPathSolver::SetState(const int position) {
absl::Status KinematicPathSolver::GetTargetPosition(int* target_position) {
RET_CHECK(initialized_)
<< "GetTargetPosition called before first observation added.";
*target_position = round(target_position_px_);
// Provide target position clamped by min/max locations.
if (target_position_px_ < min_location_) {
*target_position = min_location_;
} else if (target_position_px_ > max_location_) {
*target_position = max_location_;
} else {
*target_position = round(target_position_px_);
}
return absl::OkStatus();
}
@@ -238,6 +288,7 @@ absl::Status KinematicPathSolver::UpdateMinMaxLocation(const int min_location,
double updated_distance = max_location - min_location;
double scale_change = updated_distance / prior_distance;
current_position_px_ = current_position_px_ * scale_change;
prior_position_px_ = prior_position_px_ * scale_change;
target_position_px_ = target_position_px_ * scale_change;
max_location_ = max_location;
min_location_ = min_location;
mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver.h
+7 -2
View File
@@ -46,10 +46,12 @@ class KinematicPathSolver {
absl::Status AddObservation(int position, const uint64 time_us);
// Get the predicted position at a time.
absl::Status UpdatePrediction(const int64 time_us);
// Get the state at a time.
// Get the state at a time, as an int.
absl::Status GetState(int* position);
// Get the state at a time, as a float.
absl::Status GetState(float* position);
// Overwrite the current state value.
absl::Status SetState(const int position);
absl::Status SetState(const float position);
// Update PixelPerDegree value.
absl::Status UpdatePixelsPerDegree(const float pixels_per_degree);
// Provide the current target position of the reframe action.
@@ -66,6 +68,8 @@ class KinematicPathSolver {
// Clear any history buffer of positions that are used when
// filtering_time_window_us is set to a non-zero value.
void ClearHistory();
// Provides the change in position from last state.
absl::Status GetDeltaState(float* delta_position);
private:
// Tuning options.
@@ -77,6 +81,7 @@ class KinematicPathSolver {
float pixels_per_degree_;
// Current state values.
double current_position_px_;
double prior_position_px_;
double current_velocity_deg_per_s_;
uint64 current_time_;
// History of observations (second) and their time (first).
mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver.proto
+11 -2
View File
@@ -6,8 +6,9 @@ message KinematicOptions {
// Weighted update of new camera velocity (measurement) vs current state
// (prediction).
optional double update_rate = 1 [default = 0.5, deprecated = true];
// Max velocity (degrees per second) that the camera can move.
optional double max_velocity = 2 [default = 18];
// Max velocity (degrees per second) that the camera can move. Cannot be used
// with max_velocity_scale or max_velocity_shift.
optional double max_velocity = 2;
// Min motion (in degrees) to react for both upper and lower directions. Must
// not be set if using min_motion_to_reframe_lower and
// min_motion_to_reframe_upper.
@@ -30,4 +31,12 @@ message KinematicOptions {
optional int64 filtering_time_window_us = 7 [default = 0];
// Weighted update of average period, used for motion updates.
optional float mean_period_update_rate = 8 [default = 0.25];
// Scale factor for max velocity, to be multiplied by the distance from center
// in degrees. Cannot be used with max_velocity and must be used with
// max_velocity_shift.
optional float max_velocity_scale = 11;
// Shift factor for max velocity, to be added to the scaled distance from
// center in degrees. Cannot be used with max_velocity and must be used with
// max_velocity_scale.
optional float max_velocity_shift = 12;
}
mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver_test.cc
+82 -39
View File
@@ -36,7 +36,7 @@ TEST(KinematicPathSolverTest, FailZeroPixelsPerDegree) {
TEST(KinematicPathSolverTest, FailNotInitializedState) {
KinematicOptions options;
KinematicPathSolver solver(options, 0, 1000, 1000.0 / kWidthFieldOfView);
int state;
float state;
EXPECT_FALSE(solver.GetState(&state).ok());
}
@@ -55,13 +55,13 @@ TEST(KinematicPathSolverTest, PassNotEnoughMotionLargeImg) {
options.set_max_velocity(1000);
// Set degrees / pixel to 16.6
KinematicPathSolver solver(options, 0, 1000, 1000.0 / kWidthFieldOfView);
int state;
float state;
MP_ASSERT_OK(solver.AddObservation(500, kMicroSecInSec * 0));
// Move target by 20px / 16.6 = 1.2deg
MP_ASSERT_OK(solver.AddObservation(520, kMicroSecInSec * 1));
MP_ASSERT_OK(solver.GetState(&state));
// Expect cam to not move.
EXPECT_EQ(state, 500);
EXPECT_FLOAT_EQ(state, 500);
}
TEST(KinematicPathSolverTest, PassNotEnoughMotionSmallImg) {
@@ -72,13 +72,13 @@ TEST(KinematicPathSolverTest, PassNotEnoughMotionSmallImg) {
options.set_max_velocity(500);
// Set degrees / pixel to 8.3
KinematicPathSolver solver(options, 0, 500, 500.0 / kWidthFieldOfView);
int state;
float state;
MP_ASSERT_OK(solver.AddObservation(400, kMicroSecInSec * 0));
// Move target by 10px / 8.3 = 1.2deg
MP_ASSERT_OK(solver.AddObservation(410, kMicroSecInSec * 1));
MP_ASSERT_OK(solver.GetState(&state));
// Expect cam to not move.
EXPECT_EQ(state, 400);
EXPECT_FLOAT_EQ(state, 400);
}
TEST(KinematicPathSolverTest, PassEnoughMotionFiltered) {
@@ -90,7 +90,7 @@ TEST(KinematicPathSolverTest, PassEnoughMotionFiltered) {
options.set_filtering_time_window_us(3000000);
// Set degrees / pixel to 16.6
KinematicPathSolver solver(options, 0, 1000, 1000.0 / kWidthFieldOfView);
int state;
float state;
MP_ASSERT_OK(solver.AddObservation(500, kMicroSecInSec * 0));
// Move target by 20px / 16.6 = 1.2deg
MP_ASSERT_OK(solver.AddObservation(500, kMicroSecInSec * 1));
@@ -98,7 +98,7 @@ TEST(KinematicPathSolverTest, PassEnoughMotionFiltered) {
MP_ASSERT_OK(solver.AddObservation(500, kMicroSecInSec * 3));
MP_ASSERT_OK(solver.GetState(&state));
// Expect cam to not move.
EXPECT_EQ(state, 500);
EXPECT_FLOAT_EQ(state, 500);
}
TEST(KinematicPathSolverTest, PassEnoughMotionNotFiltered) {
@@ -110,7 +110,7 @@ TEST(KinematicPathSolverTest, PassEnoughMotionNotFiltered) {
options.set_filtering_time_window_us(0);
// Set degrees / pixel to 16.6
KinematicPathSolver solver(options, 0, 1000, 1000.0 / kWidthFieldOfView);
int state;
float state;
MP_ASSERT_OK(solver.AddObservation(500, kMicroSecInSec * 0));
// Move target by 20px / 16.6 = 1.2deg
MP_ASSERT_OK(solver.AddObservation(500, kMicroSecInSec * 1));
@@ -118,7 +118,7 @@ TEST(KinematicPathSolverTest, PassEnoughMotionNotFiltered) {
MP_ASSERT_OK(solver.AddObservation(500, kMicroSecInSec * 3));
MP_ASSERT_OK(solver.GetState(&state));
// Expect cam to not move.
EXPECT_EQ(state, 506);
EXPECT_FLOAT_EQ(state, 506.4);
}
TEST(KinematicPathSolverTest, PassEnoughMotionLargeImg) {
@@ -130,13 +130,13 @@ TEST(KinematicPathSolverTest, PassEnoughMotionLargeImg) {
options.set_max_velocity(1000);
// Set degrees / pixel to 16.6
KinematicPathSolver solver(options, 0, 1000, 1000.0 / kWidthFieldOfView);
int state;
float state;
MP_ASSERT_OK(solver.AddObservation(500, kMicroSecInSec * 0));
// Move target by 20px / 16.6 = 1.2deg
MP_ASSERT_OK(solver.AddObservation(520, kMicroSecInSec * 1));
MP_ASSERT_OK(solver.GetState(&state));
// Expect cam to move.
EXPECT_EQ(state, 520);
EXPECT_FLOAT_EQ(state, 520);
}
TEST(KinematicPathSolverTest, PassEnoughMotionSmallImg) {
@@ -148,13 +148,13 @@ TEST(KinematicPathSolverTest, PassEnoughMotionSmallImg) {
options.set_max_velocity(18);
// Set degrees / pixel to 8.3
KinematicPathSolver solver(options, 0, 500, 500.0 / kWidthFieldOfView);
int state;
float state;
MP_ASSERT_OK(solver.AddObservation(400, kMicroSecInSec * 0));
// Move target by 10px / 8.3 = 1.2deg
MP_ASSERT_OK(solver.AddObservation(410, kMicroSecInSec * 1));
MP_ASSERT_OK(solver.GetState(&state));
// Expect cam to move.
EXPECT_EQ(state, 410);
EXPECT_FLOAT_EQ(state, 410);
}
TEST(KinematicPathSolverTest, FailReframeWindowSetting) {
@@ -181,13 +181,13 @@ TEST(KinematicPathSolverTest, PassReframeWindow) {
options.set_reframe_window(0.75);
// Set degrees / pixel to 16.6
KinematicPathSolver solver(options, 0, 1000, 1000.0 / kWidthFieldOfView);
int state;
float state;
MP_ASSERT_OK(solver.AddObservation(500, kMicroSecInSec * 0));
// Move target by 20px / 16.6 = 1.2deg
MP_ASSERT_OK(solver.AddObservation(520, kMicroSecInSec * 1));
MP_ASSERT_OK(solver.GetState(&state));
// Expect cam to move 1.2-.75 deg, * 16.6 = 7.47px + 500 =
EXPECT_EQ(state, 508);
EXPECT_FLOAT_EQ(state, 507.5);
}
TEST(KinematicPathSolverTest, PassReframeWindowLowerUpper) {
@@ -202,17 +202,17 @@ TEST(KinematicPathSolverTest, PassReframeWindowLowerUpper) {
options.set_reframe_window(0.75);
// Set degrees / pixel to 16.6
KinematicPathSolver solver(options, 0, 1000, 1000.0 / kWidthFieldOfView);
int state;
float state;
MP_ASSERT_OK(solver.AddObservation(500, kMicroSecInSec * 0));
// Move target by 20px / 16.6 = 1.2deg
MP_ASSERT_OK(solver.AddObservation(520, kMicroSecInSec * 1));
MP_ASSERT_OK(solver.GetState(&state));
// Expect cam to not move
EXPECT_EQ(state, 500);
EXPECT_FLOAT_EQ(state, 500);
MP_ASSERT_OK(solver.AddObservation(480, kMicroSecInSec * 2));
MP_ASSERT_OK(solver.GetState(&state));
// Expect cam to move
EXPECT_EQ(state, 493);
EXPECT_FLOAT_EQ(state, 492.5);
}
TEST(KinematicPathSolverTest, PassCheckState) {
@@ -241,12 +241,12 @@ TEST(KinematicPathSolverTest, PassUpdateRate30FPS) {
options.set_max_update_rate(0.8);
options.set_max_velocity(18);
KinematicPathSolver solver(options, 0, 1000, 1000.0 / kWidthFieldOfView);
int state;
float state;
MP_ASSERT_OK(solver.AddObservation(500, kMicroSecInSec * 0));
MP_ASSERT_OK(solver.AddObservation(520, kMicroSecInSec * 1 / 30));
MP_ASSERT_OK(solver.GetState(&state));
// (0.033 / .25) * 20 =
EXPECT_EQ(state, 503);
EXPECT_FLOAT_EQ(state, 502.6667);
}
TEST(KinematicPathSolverTest, PassUpdateRate10FPS) {
@@ -256,12 +256,12 @@ TEST(KinematicPathSolverTest, PassUpdateRate10FPS) {
options.set_max_update_rate(0.8);
options.set_max_velocity(18);
KinematicPathSolver solver(options, 0, 1000, 1000.0 / kWidthFieldOfView);
int state;
float state;
MP_ASSERT_OK(solver.AddObservation(500, kMicroSecInSec * 0));
MP_ASSERT_OK(solver.AddObservation(520, kMicroSecInSec * 1 / 10));
MP_ASSERT_OK(solver.GetState(&state));
// (0.1 / .25) * 20 =
EXPECT_EQ(state, 508);
EXPECT_FLOAT_EQ(state, 508);
}
TEST(KinematicPathSolverTest, PassUpdateRate) {
@@ -271,7 +271,8 @@ TEST(KinematicPathSolverTest, PassUpdateRate) {
options.set_max_update_rate(1.0);
options.set_max_velocity(18);
KinematicPathSolver solver(options, 0, 1000, 1000.0 / kWidthFieldOfView);
int state, target_position;
int target_position;
float state;
MP_ASSERT_OK(solver.AddObservation(500, kMicroSecInSec * 0));
MP_ASSERT_OK(solver.GetTargetPosition(&target_position));
EXPECT_EQ(target_position, 500);
@@ -279,7 +280,7 @@ TEST(KinematicPathSolverTest, PassUpdateRate) {
MP_ASSERT_OK(solver.GetTargetPosition(&target_position));
EXPECT_EQ(target_position, 520);
MP_ASSERT_OK(solver.GetState(&state));
EXPECT_EQ(state, 505);
EXPECT_FLOAT_EQ(state, 505);
}
TEST(KinematicPathSolverTest, PassUpdateRateResolutionChange) {
@@ -289,7 +290,8 @@ TEST(KinematicPathSolverTest, PassUpdateRateResolutionChange) {
options.set_max_update_rate(1.0);
options.set_max_velocity(18);
KinematicPathSolver solver(options, 0, 1000, 1000.0 / kWidthFieldOfView);
int state, target_position;
int target_position;
float state;
MP_ASSERT_OK(solver.AddObservation(500, kMicroSecInSec * 0));
MP_ASSERT_OK(solver.GetTargetPosition(&target_position));
EXPECT_EQ(target_position, 500);
@@ -299,10 +301,10 @@ TEST(KinematicPathSolverTest, PassUpdateRateResolutionChange) {
MP_ASSERT_OK(solver.GetTargetPosition(&target_position));
EXPECT_EQ(target_position, 520 * 0.5);
MP_ASSERT_OK(solver.GetState(&state));
EXPECT_EQ(state, 253);
EXPECT_FLOAT_EQ(state, 252.5);
}
TEST(KinematicPathSolverTest, PassMaxVelocity) {
TEST(KinematicPathSolverTest, PassMaxVelocityInt) {
KinematicOptions options;
options.set_min_motion_to_reframe(1.0);
options.set_update_rate(1.0);
@@ -315,6 +317,33 @@ TEST(KinematicPathSolverTest, PassMaxVelocity) {
EXPECT_EQ(state, 600);
}
TEST(KinematicPathSolverTest, PassMaxVelocity) {
KinematicOptions options;
options.set_min_motion_to_reframe(1.0);
options.set_update_rate(1.0);
options.set_max_velocity(6);
KinematicPathSolver solver(options, 0, 1000, 1000.0 / kWidthFieldOfView);
float state;
MP_ASSERT_OK(solver.AddObservation(500, kMicroSecInSec * 0));
MP_ASSERT_OK(solver.AddObservation(1000, kMicroSecInSec * 1));
MP_ASSERT_OK(solver.GetState(&state));
EXPECT_FLOAT_EQ(state, 600);
}
TEST(KinematicPathSolverTest, PassMaxVelocityScale) {
KinematicOptions options;
options.set_min_motion_to_reframe(1.0);
options.set_update_rate(1.0);
options.set_max_velocity_scale(0.4);
options.set_max_velocity_shift(-2.0);
KinematicPathSolver solver(options, 0, 1000, 1000.0 / kWidthFieldOfView);
float state;
MP_ASSERT_OK(solver.AddObservation(500, kMicroSecInSec * 0));
MP_ASSERT_OK(solver.AddObservation(1000, kMicroSecInSec * 1));
MP_ASSERT_OK(solver.GetState(&state));
EXPECT_FLOAT_EQ(state, 666.6667);
}
TEST(KinematicPathSolverTest, PassDegPerPxChange) {
KinematicOptions options;
// Set min motion to 2deg
@@ -323,18 +352,18 @@ TEST(KinematicPathSolverTest, PassDegPerPxChange) {
options.set_max_velocity(1000);
// Set degrees / pixel to 16.6
KinematicPathSolver solver(options, 0, 1000, 1000.0 / kWidthFieldOfView);
int state;
float state;
MP_ASSERT_OK(solver.AddObservation(500, kMicroSecInSec * 0));
// Move target by 20px / 16.6 = 1.2deg
MP_ASSERT_OK(solver.AddObservation(520, kMicroSecInSec * 1));
MP_ASSERT_OK(solver.GetState(&state));
// Expect cam to not move.
EXPECT_EQ(state, 500);
EXPECT_FLOAT_EQ(state, 500);
MP_ASSERT_OK(solver.UpdatePixelsPerDegree(500.0 / kWidthFieldOfView));
MP_ASSERT_OK(solver.AddObservation(520, kMicroSecInSec * 2));
MP_ASSERT_OK(solver.GetState(&state));
// Expect cam to move.
EXPECT_EQ(state, 516);
EXPECT_FLOAT_EQ(state, 516);
}
TEST(KinematicPathSolverTest, NoTimestampSmoothing) {
@@ -344,14 +373,14 @@ TEST(KinematicPathSolverTest, NoTimestampSmoothing) {
options.set_max_velocity(6);
options.set_mean_period_update_rate(1.0);
KinematicPathSolver solver(options, 0, 1000, 1000.0 / kWidthFieldOfView);
int state;
float state;
MP_ASSERT_OK(solver.AddObservation(500, 0));
MP_ASSERT_OK(solver.AddObservation(1000, 1000000));
MP_ASSERT_OK(solver.GetState(&state));
EXPECT_EQ(state, 600);
EXPECT_FLOAT_EQ(state, 600);
MP_ASSERT_OK(solver.AddObservation(1000, 2200000));
MP_ASSERT_OK(solver.GetState(&state));
EXPECT_EQ(state, 720);
EXPECT_FLOAT_EQ(state, 720);
}
TEST(KinematicPathSolverTest, TimestampSmoothing) {
@@ -361,14 +390,14 @@ TEST(KinematicPathSolverTest, TimestampSmoothing) {
options.set_max_velocity(6);
options.set_mean_period_update_rate(0.05);
KinematicPathSolver solver(options, 0, 1000, 1000.0 / kWidthFieldOfView);
int state;
float state;
MP_ASSERT_OK(solver.AddObservation(500, 0));
MP_ASSERT_OK(solver.AddObservation(1000, 1000000));
MP_ASSERT_OK(solver.GetState(&state));
EXPECT_EQ(state, 600);
EXPECT_FLOAT_EQ(state, 600);
MP_ASSERT_OK(solver.AddObservation(1000, 2200000));
MP_ASSERT_OK(solver.GetState(&state));
EXPECT_EQ(state, 701);
EXPECT_FLOAT_EQ(state, 701);
}
TEST(KinematicPathSolverTest, PassSetPosition) {
@@ -380,16 +409,30 @@ TEST(KinematicPathSolverTest, PassSetPosition) {
options.set_max_velocity(18);
// Set degrees / pixel to 8.3
KinematicPathSolver solver(options, 0, 500, 500.0 / kWidthFieldOfView);
int state;
float state;
MP_ASSERT_OK(solver.AddObservation(400, kMicroSecInSec * 0));
// Move target by 10px / 8.3 = 1.2deg
MP_ASSERT_OK(solver.AddObservation(410, kMicroSecInSec * 1));
MP_ASSERT_OK(solver.GetState(&state));
// Expect cam to move.
EXPECT_EQ(state, 410);
EXPECT_FLOAT_EQ(state, 410);
MP_ASSERT_OK(solver.SetState(400));
MP_ASSERT_OK(solver.GetState(&state));
EXPECT_EQ(state, 400);
EXPECT_FLOAT_EQ(state, 400);
}
TEST(KinematicPathSolverTest, PassBorderTest) {
KinematicOptions options;
options.set_min_motion_to_reframe(1.0);
options.set_max_update_rate(0.25);
options.set_max_velocity_scale(0.5);
options.set_max_velocity_shift(-1.0);
KinematicPathSolver solver(options, 0, 500, 500.0 / kWidthFieldOfView);
float state;
MP_ASSERT_OK(solver.AddObservation(400, kMicroSecInSec * 0));
MP_ASSERT_OK(solver.AddObservation(800, kMicroSecInSec * 0.1));
MP_ASSERT_OK(solver.GetState(&state));
EXPECT_FLOAT_EQ(state, 404.56668);
}
} // namespace