talksik/mediapipe
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Project import generated by Copybara.

Browse Source 
GitOrigin-RevId: b695dda274aa3ac3c7d054e150bd9eb5c1285b19
This commit is contained in:
MediaPipe Team
2020-01-17 15:49:22 -08:00
committed by chris
parent 66b377c825
commit dd02df1dbe
39 changed files with 692 additions and 203 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mediapipe/examples/desktop/autoflip/quality/polynomial_regression_path_solver.cc
+11 -2
View File
@@ -52,15 +52,24 @@ struct PolynomialResidual {
const double out_;
};
// Computes the amount of delta position change along the fitted polynomial
// curve, translates the delta from being relative to the origin of the original
// dimension to being relative to the center of the original dimension, then
// regulates the delta to avoid moving camera off the frame boundaries.
float ComputeDelta(const float in, const int original_dimension,
const int output_dimension, const double a, const double b,
const double c, const double d, const double k) {
// The value `out` here represents a normalized distance between the center of
// the output window and the origin of the original window.
float out =
a * in + b * in * in + c * in * in * in + d * in * in * in * in + k;
float delta = (out - 0.5) * 2 * output_dimension;
const float max_delta = (original_dimension - output_dimension) / 2.0f;
// Translate `out` to a pixel distance between the center of the output window
// and the center of the original window. This value can be negative, 0, or
// positive.
float delta = (out - 0.5) * original_dimension;
// Make sure delta doesn't move the camera off the frame boundary.
const float max_delta = (original_dimension - output_dimension) / 2.0f;
if (delta > max_delta) {
delta = max_delta;
} else if (delta < -max_delta) {
mediapipe/examples/desktop/autoflip/quality/polynomial_regression_path_solver_test.cc
+89 -14
View File
@@ -140,7 +140,7 @@ constexpr double prediction[] = {
-24.550379, -24.06503,
};
void GenerateDataPoints(
void GenerateDataPointsFromRealVideo(
const int focus_point_frames_length,
const int prior_focus_point_frames_length,
std::vector<FocusPointFrame>* focus_point_frames,
@@ -163,14 +163,15 @@ void GenerateDataPoints(
}
}
TEST(PolynomialRegressionPathSolverTest, Success) {
TEST(PolynomialRegressionPathSolverTest, SuccessInTrackingCameraMode) {
PolynomialRegressionPathSolver solver;
std::vector<FocusPointFrame> focus_point_frames;
std::vector<FocusPointFrame> prior_focus_point_frames;
std::vector<cv::Mat> all_xforms;
GenerateDataPoints(/* focus_point_frames_length = */ 100,
/* prior_focus_point_frames_length = */ 100,
&focus_point_frames, &prior_focus_point_frames);
GenerateDataPointsFromRealVideo(/* focus_point_frames_length = */ 100,
/* prior_focus_point_frames_length = */ 100,
&focus_point_frames,
&prior_focus_point_frames);
constexpr int kFrameWidth = 200;
constexpr int kFrameHeight = 300;
constexpr int kCropWidth = 100;
@@ -185,14 +186,86 @@ TEST(PolynomialRegressionPathSolverTest, Success) {
}
}
TEST(PolynomialRegressionPathSolverTest, SuccessInStationaryCameraMode) {
PolynomialRegressionPathSolver solver;
std::vector<FocusPointFrame> focus_point_frames;
std::vector<FocusPointFrame> prior_focus_point_frames;
std::vector<cv::Mat> all_xforms;
constexpr int kFocusPointFramesLength = 100;
constexpr float kNormStationaryFocusPointX = 0.34f;
for (int i = 0; i < kFocusPointFramesLength; i++) {
FocusPoint sp;
// Add a fixed normalized focus point location.
sp.set_norm_point_x(kNormStationaryFocusPointX);
FocusPointFrame spf;
*spf.add_point() = sp;
focus_point_frames.push_back(spf);
}
constexpr int kFrameWidth = 300;
constexpr int kFrameHeight = 300;
constexpr int kCropWidth = 100;
constexpr int kCropHeight = 300;
MP_ASSERT_OK(solver.ComputeCameraPath(
focus_point_frames, prior_focus_point_frames, kFrameWidth, kFrameHeight,
kCropWidth, kCropHeight, &all_xforms));
ASSERT_EQ(all_xforms.size(), kFocusPointFramesLength);
constexpr int kExpectedShift = -48;
for (int i = 0; i < all_xforms.size(); i++) {
cv::Mat mat = all_xforms[i];
EXPECT_FLOAT_EQ(mat.at<float>(0, 2), kExpectedShift);
}
}
// Test the case where focus points are so close to boundaries that the amount
// of shifts would have moved the camera to go outside frame boundaries. In this
// case, the solver should regulate the camera position shift to keep it stay
// inside the viewport.
TEST(PolynomialRegressionPathSolverTest, SuccessWhenFocusPointCloseToBoundary) {
PolynomialRegressionPathSolver solver;
std::vector<FocusPointFrame> focus_point_frames;
std::vector<FocusPointFrame> prior_focus_point_frames;
std::vector<cv::Mat> all_xforms;
constexpr int kFocusPointFramesLength = 100;
constexpr float kNormStationaryFocusPointX = 0.99f;
for (int i = 0; i < kFocusPointFramesLength; i++) {
FocusPoint sp;
// Add a fixed normalized focus point location.
sp.set_norm_point_x(kNormStationaryFocusPointX);
FocusPointFrame spf;
*spf.add_point() = sp;
focus_point_frames.push_back(spf);
}
constexpr int kFrameWidth = 500;
constexpr int kFrameHeight = 300;
constexpr int kCropWidth = 100;
constexpr int kCropHeight = 300;
MP_ASSERT_OK(solver.ComputeCameraPath(
focus_point_frames, prior_focus_point_frames, kFrameWidth, kFrameHeight,
kCropWidth, kCropHeight, &all_xforms));
ASSERT_EQ(all_xforms.size(), kFocusPointFramesLength);
// Regulate max delta change = (500 - 100) / 2.
constexpr int kExpectedShift = 200;
for (int i = 0; i < all_xforms.size(); i++) {
cv::Mat mat = all_xforms[i];
EXPECT_FLOAT_EQ(mat.at<float>(0, 2), kExpectedShift);
}
}
TEST(PolynomialRegressionPathSolverTest, FewFramesShouldWork) {
PolynomialRegressionPathSolver solver;
std::vector<FocusPointFrame> focus_point_frames;
std::vector<FocusPointFrame> prior_focus_point_frames;
std::vector<cv::Mat> all_xforms;
GenerateDataPoints(/* focus_point_frames_length = */ 1,
/* prior_focus_point_frames_length = */ 1,
&focus_point_frames, &prior_focus_point_frames);
GenerateDataPointsFromRealVideo(/* focus_point_frames_length = */ 1,
/* prior_focus_point_frames_length = */ 1,
&focus_point_frames,
&prior_focus_point_frames);
constexpr int kFrameWidth = 200;
constexpr int kFrameHeight = 300;
constexpr int kCropWidth = 100;
@@ -208,9 +281,10 @@ TEST(PolynomialRegressionPathSolverTest, OneCurrentFrameShouldWork) {
std::vector<FocusPointFrame> focus_point_frames;
std::vector<FocusPointFrame> prior_focus_point_frames;
std::vector<cv::Mat> all_xforms;
GenerateDataPoints(/* focus_point_frames_length = */ 1,
/* prior_focus_point_frames_length = */ 0,
&focus_point_frames, &prior_focus_point_frames);
GenerateDataPointsFromRealVideo(/* focus_point_frames_length = */ 1,
/* prior_focus_point_frames_length = */ 0,
&focus_point_frames,
&prior_focus_point_frames);
constexpr int kFrameWidth = 200;
constexpr int kFrameHeight = 300;
constexpr int kCropWidth = 100;
@@ -226,9 +300,10 @@ TEST(PolynomialRegressionPathSolverTest, ZeroFrameShouldFail) {
std::vector<FocusPointFrame> focus_point_frames;
std::vector<FocusPointFrame> prior_focus_point_frames;
std::vector<cv::Mat> all_xforms;
GenerateDataPoints(/* focus_point_frames_length = */ 0,
/* prior_focus_point_frames_length = */ 0,
&focus_point_frames, &prior_focus_point_frames);
GenerateDataPointsFromRealVideo(/* focus_point_frames_length = */ 0,
/* prior_focus_point_frames_length = */ 0,
&focus_point_frames,
&prior_focus_point_frames);
constexpr int kFrameWidth = 200;
constexpr int kFrameHeight = 300;
constexpr int kCropWidth = 100;