talksik/mediapipe
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Project import generated by Copybara.

Browse Source 
GitOrigin-RevId: ec25bf2e416c3689477e82946fb69de2e53b9161
This commit is contained in:
MediaPipe Team
2021-06-10 01:38:18 -04:00
committed by chuoling
parent b48d72e43f
commit b544a314b3
32 changed files with 561 additions and 234 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mediapipe/examples/desktop/autoflip/calculators/content_zooming_calculator.cc
+38 -12
View File
@@ -37,6 +37,13 @@ constexpr char kFirstCropRect[] = "FIRST_CROP_RECT";
// (configured through option us_to_first_rect). If provided, a non-zero integer
// will allow the animated zoom to be used when the first detections arrive.
constexpr char kAnimateZoom[] = "ANIMATE_ZOOM";
// Can be used to control the maximum zoom; note that it is re-evaluated only
// upon change of input resolution. A value of 100 disables zooming and is the
// smallest allowed value. A value of 200 allows zooming such that a pixel of
// the input may cover up to four times its original area. Note that
// max_zoom_value_deg from options is always respected; MAX_ZOOM_PCT can only be
// used to limit zooming further.
constexpr char kMaxZoomFactorPercent[] = "MAX_ZOOM_FACTOR_PCT";
// Field-of-view (degrees) of the camera's x-axis (width).
// TODO: Parameterize FOV based on camera specs.
constexpr float kFieldOfView = 60;
@@ -75,11 +82,16 @@ class ContentZoomingCalculator : public CalculatorBase {
int frame_height);
// Saves state to a state-cache, if provided.
absl::Status SaveState(mediapipe::CalculatorContext* cc) const;
// Returns the factor for maximum zoom based on options and the
// kMaxZoomFactorPercent input (if present).
double GetMaxZoomFactor(mediapipe::CalculatorContext* cc) const;
// Initializes the calculator for the given frame size, creating path solvers
// and resetting history like last measured values.
absl::Status InitializeState(int frame_width, int frame_height);
absl::Status InitializeState(mediapipe::CalculatorContext* cc,
int frame_width, int frame_height);
// Adjusts state to work with an updated frame size.
absl::Status UpdateForResolutionChange(int frame_width, int frame_height);
absl::Status UpdateForResolutionChange(mediapipe::CalculatorContext* cc,
int frame_width, int frame_height);
// Returns true if we are animating to the first rect.
bool IsAnimatingToFirstRect(const Timestamp& timestamp) const;
// Builds the output rectangle when animating to the first rect.
@@ -136,6 +148,9 @@ absl::Status ContentZoomingCalculator::GetContract(
return mediapipe::UnknownErrorBuilder(MEDIAPIPE_LOC)
<< "Input VIDEO or VIDEO_SIZE must be provided.";
}
if (cc->Inputs().HasTag(kMaxZoomFactorPercent)) {
cc->Inputs().Tag(kMaxZoomFactorPercent).Set<int>();
}
if (cc->Inputs().HasTag(kSalientRegions)) {
cc->Inputs().Tag(kSalientRegions).Set<DetectionSet>();
}
@@ -330,7 +345,7 @@ absl::Status ContentZoomingCalculator::MaybeLoadState(
? cc->InputSidePackets().Tag(kStateCache).Get<StateCacheType*>()
: nullptr;
if (!state_cache || !state_cache->has_value()) {
return InitializeState(frame_width, frame_height);
return InitializeState(cc, frame_width, frame_height);
}
const ContentZoomingCalculatorState& state = state_cache->value();
@@ -350,7 +365,7 @@ absl::Status ContentZoomingCalculator::MaybeLoadState(
last_measured_y_offset_ = state.last_measured_y_offset;
MP_RETURN_IF_ERROR(UpdateAspectAndMax());
return UpdateForResolutionChange(frame_width, frame_height);
return UpdateForResolutionChange(cc, frame_width, frame_height);
}
absl::Status ContentZoomingCalculator::SaveState(
@@ -379,8 +394,20 @@ absl::Status ContentZoomingCalculator::SaveState(
return absl::OkStatus();
}
absl::Status ContentZoomingCalculator::InitializeState(int frame_width,
int frame_height) {
double ContentZoomingCalculator::GetMaxZoomFactor(
mediapipe::CalculatorContext* cc) const {
double max_zoom_value =
options_.max_zoom_value_deg() / static_cast<double>(kFieldOfView);
if (cc->Inputs().HasTag(kMaxZoomFactorPercent)) {
const double factor = std::max(
1.0, cc->Inputs().Tag(kMaxZoomFactorPercent).Get<int>() / 100.0);
max_zoom_value = std::max(max_zoom_value, 1.0 / factor);
}
return max_zoom_value;
}
absl::Status ContentZoomingCalculator::InitializeState(
mediapipe::CalculatorContext* cc, int frame_width, int frame_height) {
frame_width_ = frame_width;
frame_height_ = frame_height;
path_solver_pan_ = std::make_unique<KinematicPathSolver>(
@@ -390,8 +417,7 @@ absl::Status ContentZoomingCalculator::InitializeState(int frame_width,
options_.kinematic_options_tilt(), 0, frame_height_,
static_cast<float>(frame_height_) / kFieldOfView);
MP_RETURN_IF_ERROR(UpdateAspectAndMax());
int min_zoom_size = frame_height_ * (options_.max_zoom_value_deg() /
static_cast<double>(kFieldOfView));
int min_zoom_size = frame_height_ * GetMaxZoomFactor(cc);
path_solver_zoom_ = std::make_unique<KinematicPathSolver>(
options_.kinematic_options_zoom(), min_zoom_size,
max_frame_value_ * frame_height_,
@@ -405,7 +431,7 @@ absl::Status ContentZoomingCalculator::InitializeState(int frame_width,
}
absl::Status ContentZoomingCalculator::UpdateForResolutionChange(
int frame_width, int frame_height) {
mediapipe::CalculatorContext* cc, int frame_width, int frame_height) {
// Update state for change in input resolution.
if (frame_width_ != frame_width || frame_height_ != frame_height) {
double width_scale = frame_width / static_cast<double>(frame_width_);
@@ -419,8 +445,7 @@ absl::Status ContentZoomingCalculator::UpdateForResolutionChange(
MP_RETURN_IF_ERROR(path_solver_pan_->UpdateMinMaxLocation(0, frame_width_));
MP_RETURN_IF_ERROR(
path_solver_tilt_->UpdateMinMaxLocation(0, frame_height_));
int min_zoom_size = frame_height_ * (options_.max_zoom_value_deg() /
static_cast<double>(kFieldOfView));
int min_zoom_size = frame_height_ * GetMaxZoomFactor(cc);
MP_RETURN_IF_ERROR(path_solver_zoom_->UpdateMinMaxLocation(
min_zoom_size, max_frame_value_ * frame_height_));
MP_RETURN_IF_ERROR(path_solver_zoom_->UpdatePixelsPerDegree(
@@ -493,7 +518,8 @@ absl::Status ContentZoomingCalculator::Process(
MP_RETURN_IF_ERROR(MaybeLoadState(cc, frame_width, frame_height));
initialized_ = !options_.is_stateless();
} else {
MP_RETURN_IF_ERROR(UpdateForResolutionChange(frame_width, frame_height));
MP_RETURN_IF_ERROR(
UpdateForResolutionChange(cc, frame_width, frame_height));
}
bool only_required_found = false;
mediapipe/examples/desktop/autoflip/calculators/content_zooming_calculator_test.cc
+66 -2
View File
@@ -150,6 +150,29 @@ const char kConfigE[] = R"(
}
)";
const char kConfigF[] = R"(
calculator: "ContentZoomingCalculator"
input_stream: "VIDEO_SIZE:size"
input_stream: "DETECTIONS:detections"
input_stream: "MAX_ZOOM_FACTOR_PCT:max_zoom_factor_pct"
output_stream: "CROP_RECT:rect"
output_stream: "FIRST_CROP_RECT:first_rect"
options: {
[mediapipe.autoflip.ContentZoomingCalculatorOptions.ext]: {
max_zoom_value_deg: 0
kinematic_options_zoom {
min_motion_to_reframe: 1.2
}
kinematic_options_tilt {
min_motion_to_reframe: 1.2
}
kinematic_options_pan {
min_motion_to_reframe: 1.2
}
}
}
)";
void CheckBorder(const StaticFeatures& static_features, int width, int height,
int top_border, int bottom_border) {
ASSERT_EQ(2, static_features.border().size());
@@ -170,6 +193,7 @@ void CheckBorder(const StaticFeatures& static_features, int width, int height,
struct AddDetectionFlags {
std::optional<bool> animated_zoom;
std::optional<int> max_zoom_factor_percent;
};
void AddDetectionFrameSize(const cv::Rect_<float>& position, const int64 time,
@@ -211,6 +235,14 @@ void AddDetectionFrameSize(const cv::Rect_<float>& position, const int64 time,
mediapipe::MakePacket<bool>(flags.animated_zoom.value())
.At(Timestamp(time)));
}
if (flags.max_zoom_factor_percent.has_value()) {
runner->MutableInputs()
->Tag("MAX_ZOOM_FACTOR_PCT")
.packets.push_back(
mediapipe::MakePacket<int>(flags.max_zoom_factor_percent.value())
.At(Timestamp(time)));
}
}
void AddDetection(const cv::Rect_<float>& position, const int64 time,
@@ -259,6 +291,7 @@ TEST(ContentZoomingCalculatorTest, ZoomTest) {
CheckBorder(static_features, 1000, 1000, 495, 395);
}
#if 0
TEST(ContentZoomingCalculatorTest, ZoomTestFullPTZ) {
auto runner = ::absl::make_unique<CalculatorRunner>(
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfigD));
@@ -694,8 +727,8 @@ TEST(ContentZoomingCalculatorTest, ResolutionChangeZoomingWithCache) {
auto runner = ::absl::make_unique<CalculatorRunner>(config);
runner->MutableSidePackets()->Tag("STATE_CACHE") = MakePacket<
mediapipe::autoflip::ContentZoomingCalculatorStateCacheType*>(&cache);
AddDetectionFrameSize(cv::Rect_<float>(.4, .4, .2, .2), 1000000, 1000, 1000,
runner.get());
AddDetectionFrameSize(cv::Rect_<float>(.4, .4, .2, .2), 1000000, 1000,
1000, runner.get());
AddDetectionFrameSize(cv::Rect_<float>(.4, .4, .2, .2), 2000000, 500, 500,
runner.get());
MP_ASSERT_OK(runner->Run());
@@ -719,6 +752,36 @@ TEST(ContentZoomingCalculatorTest, MaxZoomValue) {
CheckCropRect(500, 500, 916, 916, 0,
runner->Outputs().Tag("CROP_RECT").packets);
}
#endif
TEST(ContentZoomingCalculatorTest, MaxZoomValueOverride) {
auto config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfigF);
auto* options = config.mutable_options()->MutableExtension(
ContentZoomingCalculatorOptions::ext);
options->set_max_zoom_value_deg(30);
auto runner = ::absl::make_unique<CalculatorRunner>(config);
AddDetectionFrameSize(cv::Rect_<float>(.4, .4, .2, .2), 0, 640, 480,
runner.get(), {.max_zoom_factor_percent = 133});
// Change resolution and allow more zoom, and give time to use the new limit
AddDetectionFrameSize(cv::Rect_<float>(.4, .4, .2, .2), 1000000, 1280, 720,
runner.get(), {.max_zoom_factor_percent = 166});
AddDetectionFrameSize(cv::Rect_<float>(.4, .4, .2, .2), 2000000, 1280, 720,
runner.get(), {.max_zoom_factor_percent = 166});
// Switch back to a smaller resolution with a more limited zoom
AddDetectionFrameSize(cv::Rect_<float>(.4, .4, .2, .2), 3000000, 640, 480,
runner.get(), {.max_zoom_factor_percent = 133});
MP_ASSERT_OK(runner->Run());
// Max. 133% zoomed in means min. (100/133) ~ 75% of height left: ~360
// Max. 166% zoomed in means min. (100/166) ~ 60% of height left: ~430
CheckCropRect(320, 240, 480, 360, 0,
runner->Outputs().Tag("CROP_RECT").packets);
CheckCropRect(640, 360, 769, 433, 2,
runner->Outputs().Tag("CROP_RECT").packets);
CheckCropRect(320, 240, 480, 360, 3,
runner->Outputs().Tag("CROP_RECT").packets);
}
#if 0
TEST(ContentZoomingCalculatorTest, MaxZoomOutValue) {
auto config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfigD);
auto* options = config.mutable_options()->MutableExtension(
@@ -906,6 +969,7 @@ TEST(ContentZoomingCalculatorTest, ProvidesConstantFirstRect) {
EXPECT_EQ(first_rect.height(), rect.height());
}
}
#endif
} // namespace
} // namespace autoflip
mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver.cc
+9 -11
View File
@@ -154,10 +154,18 @@ absl::Status KinematicPathSolver::AddObservation(int position,
// Time and position updates.
double delta_t = (time_us - current_time_) / 1000000.0;
// Time since last state/prediction update, smoothed by
// mean_period_update_rate.
if (mean_delta_t_ < 0) {
mean_delta_t_ = delta_t;
} else {
mean_delta_t_ = mean_delta_t_ * (1 - options_.mean_period_update_rate()) +
delta_t * options_.mean_period_update_rate();
}
// Observed velocity and then weighted update of this velocity.
double observed_velocity = delta_degs / delta_t;
double update_rate = std::min(delta_t / options_.update_rate_seconds(),
double update_rate = std::min(mean_delta_t_ / options_.update_rate_seconds(),
options_.max_update_rate());
double updated_velocity = current_velocity_deg_per_s_ * (1 - update_rate) +
observed_velocity * update_rate;
@@ -174,16 +182,6 @@ absl::Status KinematicPathSolver::UpdatePrediction(const int64 time_us) {
RET_CHECK(current_time_ < time_us)
<< "Prediction time added before a prior observation or prediction.";
// Time since last state/prediction update, smoothed by
// mean_period_update_rate.
double delta_t = (time_us - current_time_) / 1000000.0;
if (mean_delta_t_ < 0) {
mean_delta_t_ = delta_t;
} else {
mean_delta_t_ = mean_delta_t_ * (1 - options_.mean_period_update_rate()) +
delta_t * options_.mean_period_update_rate();
}
// Position update limited by min/max.
double update_position_px =
current_position_px_ +
mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver_test.cc
+34
View File
@@ -337,6 +337,40 @@ TEST(KinematicPathSolverTest, PassDegPerPxChange) {
EXPECT_EQ(state, 516);
}
TEST(KinematicPathSolverTest, NoTimestampSmoothing) {
KinematicOptions options;
options.set_min_motion_to_reframe(1.0);
options.set_update_rate(1.0);
options.set_max_velocity(6);
options.set_mean_period_update_rate(1.0);
KinematicPathSolver solver(options, 0, 1000, 1000.0 / kWidthFieldOfView);
int state;
MP_ASSERT_OK(solver.AddObservation(500, 0));
MP_ASSERT_OK(solver.AddObservation(1000, 1000000));
MP_ASSERT_OK(solver.GetState(&state));
EXPECT_EQ(state, 600);
MP_ASSERT_OK(solver.AddObservation(1000, 2200000));
MP_ASSERT_OK(solver.GetState(&state));
EXPECT_EQ(state, 720);
}
TEST(KinematicPathSolverTest, TimestampSmoothing) {
KinematicOptions options;
options.set_min_motion_to_reframe(1.0);
options.set_update_rate(1.0);
options.set_max_velocity(6);
options.set_mean_period_update_rate(0.05);
KinematicPathSolver solver(options, 0, 1000, 1000.0 / kWidthFieldOfView);
int state;
MP_ASSERT_OK(solver.AddObservation(500, 0));
MP_ASSERT_OK(solver.AddObservation(1000, 1000000));
MP_ASSERT_OK(solver.GetState(&state));
EXPECT_EQ(state, 600);
MP_ASSERT_OK(solver.AddObservation(1000, 2200000));
MP_ASSERT_OK(solver.GetState(&state));
EXPECT_EQ(state, 701);
}
} // namespace
} // namespace autoflip
} // namespace mediapipe