talksik/mediapipe
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Project import generated by Copybara.

Browse Source 
GitOrigin-RevId: ff83882955f1a1e2a043ff4e71278be9d7217bbe
This commit is contained in:
MediaPipe Team
2021-05-05 14:56:16 -04:00
committed by chuoling
parent ecb5b5f44a
commit a9b643e0f5
210 changed files with 5312 additions and 3838 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mediapipe/examples/desktop/autoflip/calculators/content_zooming_calculator.cc
+50 -29
View File
@@ -33,6 +33,10 @@ constexpr char kDetections[] = "DETECTIONS";
constexpr char kDetectedBorders[] = "BORDERS";
constexpr char kCropRect[] = "CROP_RECT";
constexpr char kFirstCropRect[] = "FIRST_CROP_RECT";
// Can be used to control whether an animated zoom should actually performed
// (configured through option us_to_first_rect). If provided, a non-zero integer
// will allow the animated zoom to be used when the first detections arrive.
constexpr char kAnimateZoom[] = "ANIMATE_ZOOM";
// Field-of-view (degrees) of the camera's x-axis (width).
// TODO: Parameterize FOV based on camera specs.
constexpr float kFieldOfView = 60;
@@ -76,10 +80,10 @@ class ContentZoomingCalculator : public CalculatorBase {
absl::Status InitializeState(int frame_width, int frame_height);
// Adjusts state to work with an updated frame size.
absl::Status UpdateForResolutionChange(int frame_width, int frame_height);
// Returns true if we are zooming to the initial rect.
bool IsZoomingToInitialRect(const Timestamp& timestamp) const;
// Builds the output rectangle when zooming to the initial rect.
absl::StatusOr<mediapipe::Rect> GetInitialZoomingRect(
// Returns true if we are animating to the first rect.
bool IsAnimatingToFirstRect(const Timestamp& timestamp) const;
// Builds the output rectangle when animating to the first rect.
absl::StatusOr<mediapipe::Rect> GetAnimationRect(
int frame_width, int frame_height, const Timestamp& timestamp) const;
// Converts bounds to tilt offset, pan offset and height.
absl::Status ConvertToPanTiltZoom(float xmin, float xmax, float ymin,
@@ -97,7 +101,10 @@ class ContentZoomingCalculator : public CalculatorBase {
std::unique_ptr<KinematicPathSolver> path_solver_tilt_;
// Are parameters initialized.
bool initialized_;
// Stores the time of the first crop rectangle.
// Stores the time of the first crop rectangle. This is used to control
// animating to it. Until a first crop rectangle was computed, it has
// the value Timestamp::Unset(). If animating is not requested, it receives
// the value Timestamp::Done() instead of the time.
Timestamp first_rect_timestamp_;
// Stores the first crop rectangle.
mediapipe::NormalizedRect first_rect_;
@@ -135,6 +142,9 @@ absl::Status ContentZoomingCalculator::GetContract(
if (cc->Inputs().HasTag(kDetections)) {
cc->Inputs().Tag(kDetections).Set<std::vector<mediapipe::Detection>>();
}
if (cc->Inputs().HasTag(kAnimateZoom)) {
cc->Inputs().Tag(kAnimateZoom).Set<bool>();
}
if (cc->Outputs().HasTag(kDetectedBorders)) {
cc->Outputs().Tag(kDetectedBorders).Set<StaticFeatures>();
}
@@ -419,10 +429,11 @@ absl::Status ContentZoomingCalculator::UpdateForResolutionChange(
return absl::OkStatus();
}
bool ContentZoomingCalculator::IsZoomingToInitialRect(
bool ContentZoomingCalculator::IsAnimatingToFirstRect(
const Timestamp& timestamp) const {
if (options_.us_to_first_rect() == 0 ||
first_rect_timestamp_ == Timestamp::Unset()) {
first_rect_timestamp_ == Timestamp::Unset() ||
first_rect_timestamp_ == Timestamp::Done()) {
return false;
}
@@ -443,10 +454,10 @@ double easeInOutQuad(double t) {
double lerp(double a, double b, double i) { return a * (1 - i) + b * i; }
} // namespace
absl::StatusOr<mediapipe::Rect> ContentZoomingCalculator::GetInitialZoomingRect(
absl::StatusOr<mediapipe::Rect> ContentZoomingCalculator::GetAnimationRect(
int frame_width, int frame_height, const Timestamp& timestamp) const {
RET_CHECK(IsZoomingToInitialRect(timestamp))
<< "Must only be called if zooming to initial rect.";
RET_CHECK(IsAnimatingToFirstRect(timestamp))
<< "Must only be called if animating to first rect.";
const int64 delta_us = (timestamp - first_rect_timestamp_).Value();
const int64 delay = options_.us_to_first_rect_delay();
@@ -538,15 +549,20 @@ absl::Status ContentZoomingCalculator::Process(
}
}
bool zooming_to_initial_rect = IsZoomingToInitialRect(cc->InputTimestamp());
const bool may_start_animation = (options_.us_to_first_rect() != 0) &&
(!cc->Inputs().HasTag(kAnimateZoom) ||
cc->Inputs().Tag(kAnimateZoom).Get<bool>());
bool is_animating = IsAnimatingToFirstRect(cc->InputTimestamp());
int offset_y, height, offset_x;
if (zooming_to_initial_rect) {
// If we are zooming to the first rect, ignore any new incoming detections.
height = last_measured_height_;
offset_x = last_measured_x_offset_;
offset_y = last_measured_y_offset_;
} else if (only_required_found) {
if (!is_animating && options_.start_zoomed_out() && !may_start_animation &&
first_rect_timestamp_ == Timestamp::Unset()) {
// If we should start zoomed out and won't be doing an animation,
// initialize the path solvers using the full frame, ignoring detections.
height = max_frame_value_ * frame_height_;
offset_x = (target_aspect_ * height) / 2;
offset_y = frame_height_ / 2;
} else if (!is_animating && only_required_found) {
// Convert bounds to tilt/zoom and in pixel coordinates.
MP_RETURN_IF_ERROR(ConvertToPanTiltZoom(xmin, xmax, ymin, ymax, &offset_y,
&offset_x, &height));
@@ -555,9 +571,9 @@ absl::Status ContentZoomingCalculator::Process(
last_measured_height_ = height;
last_measured_x_offset_ = offset_x;
last_measured_y_offset_ = offset_y;
} else if (cc->InputTimestamp().Microseconds() -
last_only_required_detection_ >=
options_.us_before_zoomout()) {
} else if (!is_animating && cc->InputTimestamp().Microseconds() -
last_only_required_detection_ >=
options_.us_before_zoomout()) {
// No only_require detections found within salient regions packets
// arriving since us_before_zoomout duration.
height = max_frame_value_ * frame_height_ +
@@ -566,7 +582,8 @@ absl::Status ContentZoomingCalculator::Process(
offset_x = (target_aspect_ * height) / 2;
offset_y = frame_height_ / 2;
} else {
// No only detection found but using last detection due to
// Either animating to the first rectangle, or
// no only detection found but using last detection due to
// duration_before_zoomout_us setting.
height = last_measured_height_;
offset_x = last_measured_x_offset_;
@@ -642,24 +659,28 @@ absl::Status ContentZoomingCalculator::Process(
.AddPacket(Adopt(features.release()).At(cc->InputTimestamp()));
}
if (first_rect_timestamp_ == Timestamp::Unset() &&
options_.us_to_first_rect() != 0) {
first_rect_timestamp_ = cc->InputTimestamp();
// Record the first crop rectangle
if (first_rect_timestamp_ == Timestamp::Unset()) {
first_rect_.set_x_center(path_offset_x / static_cast<float>(frame_width_));
first_rect_.set_width(path_height * target_aspect_ /
static_cast<float>(frame_width_));
first_rect_.set_y_center(path_offset_y / static_cast<float>(frame_height_));
first_rect_.set_height(path_height / static_cast<float>(frame_height_));
// After setting the first rectangle, check whether we should zoom to it.
zooming_to_initial_rect = IsZoomingToInitialRect(cc->InputTimestamp());
// Record the time to serve as departure point for the animation.
// If we are not allowed to start the animation, set Timestamp::Done.
first_rect_timestamp_ =
may_start_animation ? cc->InputTimestamp() : Timestamp::Done();
// After setting the first rectangle, check whether we should animate to it.
is_animating = IsAnimatingToFirstRect(cc->InputTimestamp());
}
// Transmit downstream to glcroppingcalculator.
if (cc->Outputs().HasTag(kCropRect)) {
std::unique_ptr<mediapipe::Rect> gpu_rect;
if (zooming_to_initial_rect) {
auto rect = GetInitialZoomingRect(frame_width, frame_height,
cc->InputTimestamp());
if (is_animating) {
auto rect =
GetAnimationRect(frame_width, frame_height, cc->InputTimestamp());
MP_RETURN_IF_ERROR(rect.status());
gpu_rect = absl::make_unique<mediapipe::Rect>(*rect);
} else {
mediapipe/examples/desktop/autoflip/calculators/content_zooming_calculator.proto
+9 -3
View File
@@ -19,7 +19,7 @@ package mediapipe.autoflip;
import "mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver.proto";
import "mediapipe/framework/calculator.proto";
// NextTag: 17
// NextTag: 18
message ContentZoomingCalculatorOptions {
extend mediapipe.CalculatorOptions {
optional ContentZoomingCalculatorOptions ext = 313091992;
@@ -58,9 +58,15 @@ message ContentZoomingCalculatorOptions {
// Whether to keep state between frames or to compute the final crop rect.
optional bool is_stateless = 14 [default = false];
// Duration (in MicroSeconds) for moving to the first crop rect.
// If true, on the first packet start with the camera zoomed out and then zoom
// in on the subject. If false, the camera will start zoomed in on the
// subject.
optional bool start_zoomed_out = 17 [default = false];
// Duration (in MicroSeconds) for animating to the first crop rect.
// Note that if set, takes precedence over start_zoomed_out.
optional int64 us_to_first_rect = 15 [default = 0];
// Duration (in MicroSeconds) to delay moving to the first crop rect.
// Duration (in MicroSeconds) to delay animating to the first crop rect.
// Used only if us_to_first_rect is set and is interpreted as part of the
// us_to_first_rect time budget.
optional int64 us_to_first_rect_delay = 16 [default = 0];
mediapipe/examples/desktop/autoflip/calculators/content_zooming_calculator_test.cc
+122 -1
View File
@@ -127,6 +127,29 @@ const char kConfigD[] = R"(
}
)";
const char kConfigE[] = R"(
calculator: "ContentZoomingCalculator"
input_stream: "VIDEO_SIZE:size"
input_stream: "DETECTIONS:detections"
input_stream: "ANIMATE_ZOOM:animate_zoom"
output_stream: "CROP_RECT:rect"
output_stream: "FIRST_CROP_RECT:first_rect"
options: {
[mediapipe.autoflip.ContentZoomingCalculatorOptions.ext]: {
max_zoom_value_deg: 0
kinematic_options_zoom {
min_motion_to_reframe: 1.2
}
kinematic_options_tilt {
min_motion_to_reframe: 1.2
}
kinematic_options_pan {
min_motion_to_reframe: 1.2
}
}
}
)";
void CheckBorder(const StaticFeatures& static_features, int width, int height,
int top_border, int bottom_border) {
ASSERT_EQ(2, static_features.border().size());
@@ -145,9 +168,14 @@ void CheckBorder(const StaticFeatures& static_features, int width, int height,
EXPECT_EQ(Border::BOTTOM, part.relative_position());
}
struct AddDetectionFlags {
std::optional<bool> animated_zoom;
};
void AddDetectionFrameSize(const cv::Rect_<float>& position, const int64 time,
const int width, const int height,
CalculatorRunner* runner) {
CalculatorRunner* runner,
const AddDetectionFlags& flags = {}) {
auto detections = std::make_unique<std::vector<mediapipe::Detection>>();
if (position.width > 0 && position.height > 0) {
mediapipe::Detection detection;
@@ -175,6 +203,14 @@ void AddDetectionFrameSize(const cv::Rect_<float>& position, const int64 time,
runner->MutableInputs()
->Tag("VIDEO_SIZE")
.packets.push_back(Adopt(input_size.release()).At(Timestamp(time)));
if (flags.animated_zoom.has_value()) {
runner->MutableInputs()
->Tag("ANIMATE_ZOOM")
.packets.push_back(
mediapipe::MakePacket<bool>(flags.animated_zoom.value())
.At(Timestamp(time)));
}
}
void AddDetection(const cv::Rect_<float>& position, const int64 time,
@@ -703,7 +739,33 @@ TEST(ContentZoomingCalculatorTest, MaxZoomOutValue) {
CheckCropRect(500, 500, 1000, 1000, 2,
runner->Outputs().Tag("CROP_RECT").packets);
}
TEST(ContentZoomingCalculatorTest, StartZoomedOut) {
auto config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfigD);
auto* options = config.mutable_options()->MutableExtension(
ContentZoomingCalculatorOptions::ext);
options->set_start_zoomed_out(true);
auto runner = ::absl::make_unique<CalculatorRunner>(config);
AddDetectionFrameSize(cv::Rect_<float>(.4, .4, .2, .2), 0, 1000, 1000,
runner.get());
AddDetectionFrameSize(cv::Rect_<float>(.4, .4, .2, .2), 400000, 1000, 1000,
runner.get());
AddDetectionFrameSize(cv::Rect_<float>(.4, .4, .2, .2), 800000, 1000, 1000,
runner.get());
AddDetectionFrameSize(cv::Rect_<float>(.4, .4, .2, .2), 1000000, 1000, 1000,
runner.get());
MP_ASSERT_OK(runner->Run());
CheckCropRect(500, 500, 1000, 1000, 0,
runner->Outputs().Tag("CROP_RECT").packets);
CheckCropRect(500, 500, 880, 880, 1,
runner->Outputs().Tag("CROP_RECT").packets);
CheckCropRect(500, 500, 760, 760, 2,
runner->Outputs().Tag("CROP_RECT").packets);
CheckCropRect(500, 500, 655, 655, 3,
runner->Outputs().Tag("CROP_RECT").packets);
}
TEST(ContentZoomingCalculatorTest, AnimateToFirstRect) {
auto config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfigD);
auto* options = config.mutable_options()->MutableExtension(
ContentZoomingCalculatorOptions::ext);
@@ -733,6 +795,65 @@ TEST(ContentZoomingCalculatorTest, StartZoomedOut) {
runner->Outputs().Tag("CROP_RECT").packets);
}
TEST(ContentZoomingCalculatorTest, CanControlAnimation) {
auto config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfigE);
auto* options = config.mutable_options()->MutableExtension(
ContentZoomingCalculatorOptions::ext);
options->set_start_zoomed_out(true);
options->set_us_to_first_rect(1000000);
options->set_us_to_first_rect_delay(500000);
auto runner = ::absl::make_unique<CalculatorRunner>(config);
// Request the animation for the first frame.
AddDetectionFrameSize(cv::Rect_<float>(.4, .4, .2, .2), 0, 1000, 1000,
runner.get(), {.animated_zoom = true});
// We now stop requesting animated zoom and expect the already started
// animation run to completion. This tests that the zoom in continues in the
// call when it was started in the Meet greenroom.
AddDetectionFrameSize(cv::Rect_<float>(.4, .4, .2, .2), 400000, 1000, 1000,
runner.get(), {.animated_zoom = false});
AddDetectionFrameSize(cv::Rect_<float>(.4, .4, .2, .2), 800000, 1000, 1000,
runner.get(), {.animated_zoom = false});
AddDetectionFrameSize(cv::Rect_<float>(.4, .4, .2, .2), 1000000, 1000, 1000,
runner.get(), {.animated_zoom = false});
AddDetectionFrameSize(cv::Rect_<float>(.4, .4, .2, .2), 1500000, 1000, 1000,
runner.get(), {.animated_zoom = false});
MP_ASSERT_OK(runner->Run());
CheckCropRect(500, 500, 1000, 1000, 0,
runner->Outputs().Tag("CROP_RECT").packets);
CheckCropRect(500, 500, 1000, 1000, 1,
runner->Outputs().Tag("CROP_RECT").packets);
CheckCropRect(500, 500, 470, 470, 2,
runner->Outputs().Tag("CROP_RECT").packets);
CheckCropRect(500, 500, 222, 222, 3,
runner->Outputs().Tag("CROP_RECT").packets);
CheckCropRect(500, 500, 222, 222, 4,
runner->Outputs().Tag("CROP_RECT").packets);
}
TEST(ContentZoomingCalculatorTest, DoesNotAnimateIfDisabledViaInput) {
auto config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfigE);
auto* options = config.mutable_options()->MutableExtension(
ContentZoomingCalculatorOptions::ext);
options->set_start_zoomed_out(true);
options->set_us_to_first_rect(1000000);
options->set_us_to_first_rect_delay(500000);
auto runner = ::absl::make_unique<CalculatorRunner>(config);
// Disable the animation already for the first frame.
AddDetectionFrameSize(cv::Rect_<float>(.4, .4, .2, .2), 0, 1000, 1000,
runner.get(), {.animated_zoom = false});
AddDetectionFrameSize(cv::Rect_<float>(.4, .4, .2, .2), 400000, 1000, 1000,
runner.get(), {.animated_zoom = false});
AddDetectionFrameSize(cv::Rect_<float>(.4, .4, .2, .2), 800000, 1000, 1000,
runner.get(), {.animated_zoom = false});
MP_ASSERT_OK(runner->Run());
CheckCropRect(500, 500, 1000, 1000, 0,
runner->Outputs().Tag("CROP_RECT").packets);
CheckCropRect(500, 500, 880, 880, 1,
runner->Outputs().Tag("CROP_RECT").packets);
CheckCropRect(500, 500, 760, 760, 2,
runner->Outputs().Tag("CROP_RECT").packets);
}
TEST(ContentZoomingCalculatorTest, ProvidesZeroSizeFirstRectWithoutDetections) {
auto config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfigD);
auto runner = ::absl::make_unique<CalculatorRunner>(config);
mediapipe/examples/desktop/autoflip/calculators/face_box_adjuster_calculator.proto
+6
View File
@@ -47,4 +47,10 @@ message FaceBoxAdjusterCalculatorOptions {
// and height respectively.
optional float ipd_face_box_width_ratio = 6 [default = 0.5566];
optional float ipd_face_box_height_ratio = 7 [default = 0.3131];
// The max look up angle before considering the eye distance unstable.
optional float max_head_tilt_angle_deg = 8 [default = 12.0];
// The max amount of time to use an old eye distance when the face look angle
// is unstable.
optional int32 max_facesize_history_us = 9 [default = 8000000];
}
mediapipe/examples/desktop/autoflip/calculators/scene_cropping_calculator_test.cc
+1 -2
View File
@@ -345,8 +345,7 @@ TEST(SceneCroppingCalculatorTest, ChecksPriorFrameBufferSize) {
TEST(SceneCroppingCalculatorTest, ChecksDebugConfigWithoutCroppedFrame) {
const CalculatorGraphConfig::Node config =
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(absl::Substitute(
kDebugConfigNoCroppedFrame, kTargetWidth, kTargetHeight,
kTargetSizeType, 0, kPriorFrameBufferSize));
kDebugConfigNoCroppedFrame, kTargetWidth, kTargetHeight));
auto runner = absl::make_unique<CalculatorRunner>(config);
const auto status = runner->Run();
EXPECT_FALSE(status.ok());
mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver.cc
+1 -1
View File
@@ -220,7 +220,7 @@ absl::Status KinematicPathSolver::GetTargetPosition(int* target_position) {
absl::Status KinematicPathSolver::UpdatePixelsPerDegree(
const float pixels_per_degree) {
RET_CHECK_GT(pixels_per_degree_, 0)
RET_CHECK_GT(pixels_per_degree, 0)
<< "pixels_per_degree must be larger than 0.";
pixels_per_degree_ = pixels_per_degree;
return absl::OkStatus();
mediapipe/examples/desktop/autoflip/subgraph/face_detection_subgraph.pbtxt
+2 -15
View File
@@ -38,7 +38,7 @@ node {
output_stream: "TENSORS:detection_tensors"
options: {
[mediapipe.TfLiteInferenceCalculatorOptions.ext] {
model_path: "mediapipe/models/face_detection_back.tflite"
model_path: "mediapipe/modules/face_detection/face_detection_back.tflite"
}
}
}
@@ -111,26 +111,13 @@ node {
}
}
# Maps detection label IDs to the corresponding label text ("Face"). The label
# map is provided in the label_map_path option.
node {
calculator: "DetectionLabelIdToTextCalculator"
input_stream: "filtered_detections"
output_stream: "labeled_detections"
options: {
[mediapipe.DetectionLabelIdToTextCalculatorOptions.ext] {
label_map_path: "mediapipe/models/face_detection_back_labelmap.txt"
}
}
}
# Adjusts detection locations (already normalized to [0.f, 1.f]) on the
# letterboxed image (after image transformation with the FIT scale mode) to the
# corresponding locations on the same image with the letterbox removed (the
# input image to the graph before image transformation).
node {
calculator: "DetectionLetterboxRemovalCalculator"
input_stream: "DETECTIONS:labeled_detections"
input_stream: "DETECTIONS:filtered_detections"
input_stream: "LETTERBOX_PADDING:letterbox_padding"
output_stream: "DETECTIONS:output_detections"
}
mediapipe/examples/desktop/upper_body_pose_tracking/BUILD
-34
View File
@@ -1,34 +0,0 @@
# Copyright 2020 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
licenses(["notice"])
package(default_visibility = ["//mediapipe/examples:__subpackages__"])
cc_binary(
name = "upper_body_pose_tracking_cpu",
deps = [
"//mediapipe/examples/desktop:demo_run_graph_main",
"//mediapipe/graphs/pose_tracking:upper_body_pose_tracking_cpu_deps",
],
)
# Linux only
cc_binary(
name = "upper_body_pose_tracking_gpu",
deps = [
"//mediapipe/examples/desktop:demo_run_graph_main_gpu",
"//mediapipe/graphs/pose_tracking:upper_body_pose_tracking_gpu_deps",
],
)