talksik/mediapipe
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Project import generated by Copybara.

Browse Source 
GitOrigin-RevId: 6e4aff1cc351be3ae4537b677f36d139ee50ce09
This commit is contained in:
MediaPipe Team
2021-03-25 22:09:18 -04:00
committed by chuoling
parent a92cff7a60
commit 7c331ad58b
175 changed files with 4804 additions and 1325 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mediapipe/examples/coral/BUILD
+2 -1
View File
@@ -27,13 +27,14 @@ cc_library(
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/formats:image_frame_opencv",
"//mediapipe/framework/port:commandlineflags",
"//mediapipe/framework/port:file_helpers",
"//mediapipe/framework/port:opencv_highgui",
"//mediapipe/framework/port:opencv_imgproc",
"//mediapipe/framework/port:opencv_video",
"//mediapipe/framework/port:parse_text_proto",
"//mediapipe/framework/port:status",
"@com_google_absl//absl/flags:flag",
"@com_google_absl//absl/flags:parse",
],
)
mediapipe/examples/coral/Dockerfile
+1 -1
View File
@@ -62,7 +62,7 @@ COPY . /mediapipe/
# Install bazel
# Please match the current MediaPipe Bazel requirements according to docs.
ARG BAZEL_VERSION=3.4.1
ARG BAZEL_VERSION=3.7.2
RUN mkdir /bazel && \
wget --no-check-certificate -O /bazel/installer.sh "https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh" && \
wget --no-check-certificate -O /bazel/LICENSE.txt "https://raw.githubusercontent.com/bazelbuild/bazel/master/LICENSE" && \
mediapipe/examples/coral/demo_run_graph_main.cc
+11 -11
View File
@@ -15,10 +15,11 @@
// An example of sending OpenCV webcam frames into a MediaPipe graph.
#include <cstdlib>
#include "absl/flags/flag.h"
#include "absl/flags/parse.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/formats/image_frame_opencv.h"
#include "mediapipe/framework/port/commandlineflags.h"
#include "mediapipe/framework/port/file_helpers.h"
#include "mediapipe/framework/port/opencv_highgui_inc.h"
#include "mediapipe/framework/port/opencv_imgproc_inc.h"
@@ -30,15 +31,14 @@ constexpr char kInputStream[] = "input_video";
constexpr char kOutputStream[] = "output_video";
constexpr char kWindowName[] = "MediaPipe";
DEFINE_string(
calculator_graph_config_file, "",
"Name of file containing text format CalculatorGraphConfig proto.");
DEFINE_string(input_video_path, "",
"Full path of video to load. "
"If not provided, attempt to use a webcam.");
DEFINE_string(output_video_path, "",
"Full path of where to save result (.mp4 only). "
"If not provided, show result in a window.");
ABSL_FLAG(std::string, calculator_graph_config_file, "",
"Name of file containing text format CalculatorGraphConfig proto.");
ABSL_FLAG(std::string, input_video_path, "",
"Full path of video to load. "
"If not provided, attempt to use a webcam.");
ABSL_FLAG(std::string, output_video_path, "",
"Full path of where to save result (.mp4 only). "
"If not provided, show result in a window.");
absl::Status RunMPPGraph() {
std::string calculator_graph_config_contents;
@@ -143,7 +143,7 @@ absl::Status RunMPPGraph() {
int main(int argc, char** argv) {
google::InitGoogleLogging(argv[0]);
gflags::ParseCommandLineFlags(&argc, &argv, true);
absl::ParseCommandLine(argc, argv);
absl::Status run_status = RunMPPGraph();
if (!run_status.ok()) {
LOG(ERROR) << "Failed to run the graph: " << run_status.message();
mediapipe/examples/desktop/BUILD
+6 -3
View File
@@ -23,13 +23,14 @@ cc_library(
srcs = ["simple_run_graph_main.cc"],
deps = [
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/port:commandlineflags",
"//mediapipe/framework/port:file_helpers",
"//mediapipe/framework/port:map_util",
"//mediapipe/framework/port:parse_text_proto",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"//mediapipe/framework/port:statusor",
"@com_google_absl//absl/flags:flag",
"@com_google_absl//absl/flags:parse",
"@com_google_absl//absl/strings",
],
)
@@ -41,13 +42,14 @@ cc_library(
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/formats:image_frame_opencv",
"//mediapipe/framework/port:commandlineflags",
"//mediapipe/framework/port:file_helpers",
"//mediapipe/framework/port:opencv_highgui",
"//mediapipe/framework/port:opencv_imgproc",
"//mediapipe/framework/port:opencv_video",
"//mediapipe/framework/port:parse_text_proto",
"//mediapipe/framework/port:status",
"@com_google_absl//absl/flags:flag",
"@com_google_absl//absl/flags:parse",
],
)
@@ -62,7 +64,6 @@ cc_library(
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/formats:image_frame_opencv",
"//mediapipe/framework/port:commandlineflags",
"//mediapipe/framework/port:file_helpers",
"//mediapipe/framework/port:opencv_highgui",
"//mediapipe/framework/port:opencv_imgproc",
@@ -72,5 +73,7 @@ cc_library(
"//mediapipe/gpu:gl_calculator_helper",
"//mediapipe/gpu:gpu_buffer",
"//mediapipe/gpu:gpu_shared_data_internal",
"@com_google_absl//absl/flags:flag",
"@com_google_absl//absl/flags:parse",
],
)
mediapipe/examples/desktop/autoflip/calculators/BUILD
+16 -4
View File
@@ -54,18 +54,27 @@ mediapipe_cc_proto_library(
deps = [":border_detection_calculator_proto"],
)
cc_library(
name = "content_zooming_calculator_state",
hdrs = ["content_zooming_calculator_state.h"],
deps = [
"//mediapipe/examples/desktop/autoflip/quality:kinematic_path_solver",
"//mediapipe/framework:timestamp",
"//mediapipe/framework/formats:rect_cc_proto",
],
)
cc_library(
name = "content_zooming_calculator",
srcs = ["content_zooming_calculator.cc"],
deps = [
":content_zooming_calculator_cc_proto",
":content_zooming_calculator_state",
"//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto",
"//mediapipe/examples/desktop/autoflip/quality:kinematic_path_solver",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/formats:detection_cc_proto",
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/formats:location_data_cc_proto",
"//mediapipe/framework/formats:rect_cc_proto",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
],
@@ -88,7 +97,9 @@ mediapipe_cc_proto_library(
"//mediapipe/examples/desktop/autoflip/quality:kinematic_path_solver_cc_proto",
"//mediapipe/framework:calculator_cc_proto",
],
visibility = ["//mediapipe/examples:__subpackages__"],
visibility = [
"//mediapipe/examples:__subpackages__",
],
deps = [
":content_zooming_calculator_proto",
],
@@ -127,6 +138,7 @@ cc_test(
deps = [
":content_zooming_calculator",
":content_zooming_calculator_cc_proto",
":content_zooming_calculator_state",
"//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto",
"//mediapipe/examples/desktop/autoflip/quality:kinematic_path_solver",
"//mediapipe/framework:calculator_framework",
@@ -368,7 +380,6 @@ cc_test(
"//mediapipe/framework/deps:file_path",
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/formats:image_frame_opencv",
"//mediapipe/framework/port:commandlineflags",
"//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:opencv_core",
"//mediapipe/framework/port:opencv_imgcodecs",
@@ -376,6 +387,7 @@ cc_test(
"//mediapipe/framework/port:parse_text_proto",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"@com_google_absl//absl/flags:flag",
"@com_google_absl//absl/strings",
],
)
mediapipe/examples/desktop/autoflip/calculators/content_zooming_calculator.cc
+266 -60
View File
@@ -17,12 +17,11 @@
#include "mediapipe/examples/desktop/autoflip/autoflip_messages.pb.h"
#include "mediapipe/examples/desktop/autoflip/calculators/content_zooming_calculator.pb.h"
#include "mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver.h"
#include "mediapipe/examples/desktop/autoflip/calculators/content_zooming_calculator_state.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/detection.pb.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/formats/location_data.pb.h"
#include "mediapipe/framework/formats/rect.pb.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/framework/port/status_builder.h"
@@ -33,12 +32,18 @@ constexpr char kSalientRegions[] = "SALIENT_REGIONS";
constexpr char kDetections[] = "DETECTIONS";
constexpr char kDetectedBorders[] = "BORDERS";
constexpr char kCropRect[] = "CROP_RECT";
constexpr char kFirstCropRect[] = "FIRST_CROP_RECT";
// Field-of-view (degrees) of the camera's x-axis (width).
// TODO: Parameterize FOV based on camera specs.
constexpr float kFieldOfView = 60;
// A pointer to a ContentZoomingCalculatorStateCacheType in a side packet.
// Used to save state on Close and load state on Open in a new graph.
// Can be used to preserve state between graphs.
constexpr char kStateCache[] = "STATE_CACHE";
namespace mediapipe {
namespace autoflip {
using StateCacheType = ContentZoomingCalculatorStateCacheType;
// Content zooming calculator zooms in on content when a detection has
// "only_required" set true or any raw detection input. It does this by
@@ -49,8 +54,7 @@ namespace autoflip {
// include mobile makeover and autofliplive face reframing.
class ContentZoomingCalculator : public CalculatorBase {
public:
ContentZoomingCalculator()
: initialized_(false), last_only_required_detection_(0) {}
ContentZoomingCalculator() : initialized_(false) {}
~ContentZoomingCalculator() override {}
ContentZoomingCalculator(const ContentZoomingCalculator&) = delete;
ContentZoomingCalculator& operator=(const ContentZoomingCalculator&) = delete;
@@ -58,8 +62,25 @@ class ContentZoomingCalculator : public CalculatorBase {
static absl::Status GetContract(mediapipe::CalculatorContract* cc);
absl::Status Open(mediapipe::CalculatorContext* cc) override;
absl::Status Process(mediapipe::CalculatorContext* cc) override;
absl::Status Close(mediapipe::CalculatorContext* cc) override;
private:
// Tries to load state from a state-cache, if provided. Fallsback to
// initializing state if no cache or no value in the cache are available.
absl::Status MaybeLoadState(mediapipe::CalculatorContext* cc, int frame_width,
int frame_height);
// Saves state to a state-cache, if provided.
absl::Status SaveState(mediapipe::CalculatorContext* cc) const;
// Initializes the calculator for the given frame size, creating path solvers
// and resetting history like last measured values.
absl::Status InitializeState(int frame_width, int frame_height);
// Adjusts state to work with an updated frame size.
absl::Status UpdateForResolutionChange(int frame_width, int frame_height);
// Returns true if we are zooming to the initial rect.
bool IsZoomingToInitialRect(const Timestamp& timestamp) const;
// Builds the output rectangle when zooming to the initial rect.
absl::StatusOr<mediapipe::Rect> GetInitialZoomingRect(
int frame_width, int frame_height, const Timestamp& timestamp) const;
// Converts bounds to tilt offset, pan offset and height.
absl::Status ConvertToPanTiltZoom(float xmin, float xmax, float ymin,
float ymax, int* tilt_offset,
@@ -76,6 +97,10 @@ class ContentZoomingCalculator : public CalculatorBase {
std::unique_ptr<KinematicPathSolver> path_solver_tilt_;
// Are parameters initialized.
bool initialized_;
// Stores the time of the first crop rectangle.
Timestamp first_rect_timestamp_;
// Stores the first crop rectangle.
mediapipe::NormalizedRect first_rect_;
// Stores the time of the last "only_required" input.
int64 last_only_required_detection_;
// Rect values of last message with detection(s).
@@ -116,6 +141,12 @@ absl::Status ContentZoomingCalculator::GetContract(
if (cc->Outputs().HasTag(kCropRect)) {
cc->Outputs().Tag(kCropRect).Set<mediapipe::Rect>();
}
if (cc->Outputs().HasTag(kFirstCropRect)) {
cc->Outputs().Tag(kFirstCropRect).Set<mediapipe::NormalizedRect>();
}
if (cc->InputSidePackets().HasTag(kStateCache)) {
cc->InputSidePackets().Tag(kStateCache).Set<StateCacheType*>();
}
return absl::OkStatus();
}
@@ -135,6 +166,13 @@ absl::Status ContentZoomingCalculator::Open(mediapipe::CalculatorContext* cc) {
return absl::OkStatus();
}
absl::Status ContentZoomingCalculator::Close(mediapipe::CalculatorContext* cc) {
if (initialized_) {
MP_RETURN_IF_ERROR(SaveState(cc));
}
return absl::OkStatus();
}
absl::Status ContentZoomingCalculator::ConvertToPanTiltZoom(
float xmin, float xmax, float ymin, float ymax, int* tilt_offset,
int* pan_offset, int* height) {
@@ -275,39 +313,89 @@ absl::Status ContentZoomingCalculator::UpdateAspectAndMax() {
return absl::OkStatus();
}
absl::Status ContentZoomingCalculator::Process(
mediapipe::CalculatorContext* cc) {
// For async subgraph support, return on empty video size packets.
if (cc->Inputs().HasTag(kVideoSize) &&
cc->Inputs().Tag(kVideoSize).IsEmpty()) {
absl::Status ContentZoomingCalculator::MaybeLoadState(
mediapipe::CalculatorContext* cc, int frame_width, int frame_height) {
const auto* state_cache =
cc->InputSidePackets().HasTag(kStateCache)
? cc->InputSidePackets().Tag(kStateCache).Get<StateCacheType*>()
: nullptr;
if (!state_cache || !state_cache->has_value()) {
return InitializeState(frame_width, frame_height);
}
const ContentZoomingCalculatorState& state = state_cache->value();
frame_width_ = state.frame_width;
frame_height_ = state.frame_height;
path_solver_pan_ =
std::make_unique<KinematicPathSolver>(state.path_solver_pan);
path_solver_tilt_ =
std::make_unique<KinematicPathSolver>(state.path_solver_tilt);
path_solver_zoom_ =
std::make_unique<KinematicPathSolver>(state.path_solver_zoom);
first_rect_timestamp_ = state.first_rect_timestamp;
first_rect_ = state.first_rect;
last_only_required_detection_ = state.last_only_required_detection;
last_measured_height_ = state.last_measured_height;
last_measured_x_offset_ = state.last_measured_x_offset;
last_measured_y_offset_ = state.last_measured_y_offset;
MP_RETURN_IF_ERROR(UpdateAspectAndMax());
return UpdateForResolutionChange(frame_width, frame_height);
}
absl::Status ContentZoomingCalculator::SaveState(
mediapipe::CalculatorContext* cc) const {
auto* state_cache =
cc->InputSidePackets().HasTag(kStateCache)
? cc->InputSidePackets().Tag(kStateCache).Get<StateCacheType*>()
: nullptr;
if (!state_cache) {
return absl::OkStatus();
}
int frame_width, frame_height;
MP_RETURN_IF_ERROR(GetVideoResolution(cc, &frame_width, &frame_height));
// Init on first call.
if (!initialized_) {
frame_width_ = frame_width;
frame_height_ = frame_height;
path_solver_pan_ = std::make_unique<KinematicPathSolver>(
options_.kinematic_options_pan(), 0, frame_width_,
static_cast<float>(frame_width_) / kFieldOfView);
path_solver_tilt_ = std::make_unique<KinematicPathSolver>(
options_.kinematic_options_tilt(), 0, frame_height_,
static_cast<float>(frame_height_) / kFieldOfView);
MP_RETURN_IF_ERROR(UpdateAspectAndMax());
int min_zoom_size = frame_height_ * (options_.max_zoom_value_deg() /
static_cast<double>(kFieldOfView));
path_solver_zoom_ = std::make_unique<KinematicPathSolver>(
options_.kinematic_options_zoom(), min_zoom_size,
max_frame_value_ * frame_height_,
static_cast<float>(frame_height_) / kFieldOfView);
last_measured_height_ = max_frame_value_ * frame_height_;
last_measured_x_offset_ = target_aspect_ * frame_width_;
last_measured_y_offset_ = frame_width_ / 2;
initialized_ = true;
}
*state_cache = ContentZoomingCalculatorState{
.frame_height = frame_height_,
.frame_width = frame_width_,
.path_solver_zoom = *path_solver_zoom_,
.path_solver_pan = *path_solver_pan_,
.path_solver_tilt = *path_solver_tilt_,
.first_rect_timestamp = first_rect_timestamp_,
.first_rect = first_rect_,
.last_only_required_detection = last_only_required_detection_,
.last_measured_height = last_measured_height_,
.last_measured_x_offset = last_measured_x_offset_,
.last_measured_y_offset = last_measured_y_offset_,
};
return absl::OkStatus();
}
absl::Status ContentZoomingCalculator::InitializeState(int frame_width,
int frame_height) {
frame_width_ = frame_width;
frame_height_ = frame_height;
path_solver_pan_ = std::make_unique<KinematicPathSolver>(
options_.kinematic_options_pan(), 0, frame_width_,
static_cast<float>(frame_width_) / kFieldOfView);
path_solver_tilt_ = std::make_unique<KinematicPathSolver>(
options_.kinematic_options_tilt(), 0, frame_height_,
static_cast<float>(frame_height_) / kFieldOfView);
MP_RETURN_IF_ERROR(UpdateAspectAndMax());
int min_zoom_size = frame_height_ * (options_.max_zoom_value_deg() /
static_cast<double>(kFieldOfView));
path_solver_zoom_ = std::make_unique<KinematicPathSolver>(
options_.kinematic_options_zoom(), min_zoom_size,
max_frame_value_ * frame_height_,
static_cast<float>(frame_height_) / kFieldOfView);
first_rect_timestamp_ = Timestamp::Unset();
last_only_required_detection_ = 0;
last_measured_height_ = max_frame_value_ * frame_height_;
last_measured_x_offset_ = target_aspect_ * frame_width_;
last_measured_y_offset_ = frame_width_ / 2;
return absl::OkStatus();
}
absl::Status ContentZoomingCalculator::UpdateForResolutionChange(
int frame_width, int frame_height) {
// Update state for change in input resolution.
if (frame_width_ != frame_width || frame_height_ != frame_height) {
double width_scale = frame_width / static_cast<double>(frame_width_);
@@ -328,6 +416,74 @@ absl::Status ContentZoomingCalculator::Process(
MP_RETURN_IF_ERROR(path_solver_zoom_->UpdatePixelsPerDegree(
static_cast<float>(frame_height_) / kFieldOfView));
}
return absl::OkStatus();
}
bool ContentZoomingCalculator::IsZoomingToInitialRect(
const Timestamp& timestamp) const {
if (options_.us_to_first_rect() == 0 ||
first_rect_timestamp_ == Timestamp::Unset()) {
return false;
}
const int64 delta_us = (timestamp - first_rect_timestamp_).Value();
return (0 <= delta_us && delta_us <= options_.us_to_first_rect());
}
namespace {
double easeInQuad(double t) { return t * t; }
double easeOutQuad(double t) { return -1 * t * (t - 2); }
double easeInOutQuad(double t) {
if (t < 0.5) {
return easeInQuad(t * 2) * 0.5;
} else {
return easeOutQuad(t * 2 - 1) * 0.5 + 0.5;
}
}
double lerp(double a, double b, double i) { return a * (1 - i) + b * i; }
} // namespace
absl::StatusOr<mediapipe::Rect> ContentZoomingCalculator::GetInitialZoomingRect(
int frame_width, int frame_height, const Timestamp& timestamp) const {
RET_CHECK(IsZoomingToInitialRect(timestamp))
<< "Must only be called if zooming to initial rect.";
const int64 delta_us = (timestamp - first_rect_timestamp_).Value();
const int64 delay = options_.us_to_first_rect_delay();
const double interpolation = easeInOutQuad(std::max(
0.0, (delta_us - delay) /
static_cast<double>(options_.us_to_first_rect() - delay)));
const double x_center = lerp(0.5, first_rect_.x_center(), interpolation);
const double y_center = lerp(0.5, first_rect_.y_center(), interpolation);
const double width = lerp(1.0, first_rect_.width(), interpolation);
const double height = lerp(1.0, first_rect_.height(), interpolation);
mediapipe::Rect gpu_rect;
gpu_rect.set_x_center(x_center * frame_width);
gpu_rect.set_width(width * frame_width);
gpu_rect.set_y_center(y_center * frame_height);
gpu_rect.set_height(height * frame_height);
return gpu_rect;
}
absl::Status ContentZoomingCalculator::Process(
mediapipe::CalculatorContext* cc) {
// For async subgraph support, return on empty video size packets.
if (cc->Inputs().HasTag(kVideoSize) &&
cc->Inputs().Tag(kVideoSize).IsEmpty()) {
return absl::OkStatus();
}
int frame_width, frame_height;
MP_RETURN_IF_ERROR(GetVideoResolution(cc, &frame_width, &frame_height));
// Init on first call or re-init always if configured to be stateless.
if (!initialized_) {
MP_RETURN_IF_ERROR(MaybeLoadState(cc, frame_width, frame_height));
initialized_ = !options_.is_stateless();
} else {
MP_RETURN_IF_ERROR(UpdateForResolutionChange(frame_width, frame_height));
}
bool only_required_found = false;
@@ -348,31 +504,52 @@ absl::Status ContentZoomingCalculator::Process(
if (cc->Inputs().HasTag(kDetections)) {
if (cc->Inputs().Tag(kDetections).IsEmpty()) {
auto default_rect = absl::make_unique<mediapipe::Rect>();
default_rect->set_x_center(frame_width_ / 2);
default_rect->set_y_center(frame_height_ / 2);
default_rect->set_width(frame_width_);
default_rect->set_height(frame_height_);
cc->Outputs().Tag(kCropRect).Add(default_rect.release(),
Timestamp(cc->InputTimestamp()));
return absl::OkStatus();
}
auto raw_detections =
cc->Inputs().Tag(kDetections).Get<std::vector<mediapipe::Detection>>();
for (const auto& detection : raw_detections) {
only_required_found = true;
MP_RETURN_IF_ERROR(UpdateRanges(
detection, options_.detection_shift_vertical(),
options_.detection_shift_horizontal(), &xmin, &xmax, &ymin, &ymax));
if (last_only_required_detection_ == 0) {
// If no detections are available and we never had any,
// simply return the full-image rectangle as crop-rect.
if (cc->Outputs().HasTag(kCropRect)) {
auto default_rect = absl::make_unique<mediapipe::Rect>();
default_rect->set_x_center(frame_width_ / 2);
default_rect->set_y_center(frame_height_ / 2);
default_rect->set_width(frame_width_);
default_rect->set_height(frame_height_);
cc->Outputs().Tag(kCropRect).Add(default_rect.release(),
Timestamp(cc->InputTimestamp()));
}
// Also provide a first crop rect: in this case a zero-sized one.
if (cc->Outputs().HasTag(kFirstCropRect)) {
cc->Outputs()
.Tag(kFirstCropRect)
.Add(new mediapipe::NormalizedRect(),
Timestamp(cc->InputTimestamp()));
}
return absl::OkStatus();
}
} else {
auto raw_detections = cc->Inputs()
.Tag(kDetections)
.Get<std::vector<mediapipe::Detection>>();
for (const auto& detection : raw_detections) {
only_required_found = true;
MP_RETURN_IF_ERROR(UpdateRanges(
detection, options_.detection_shift_vertical(),
options_.detection_shift_horizontal(), &xmin, &xmax, &ymin, &ymax));
}
}
}
// Convert bounds to tilt/zoom and in pixel coordinates.
int offset_y, height, offset_x;
MP_RETURN_IF_ERROR(ConvertToPanTiltZoom(xmin, xmax, ymin, ymax, &offset_y,
&offset_x, &height));
bool zooming_to_initial_rect = IsZoomingToInitialRect(cc->InputTimestamp());
if (only_required_found) {
int offset_y, height, offset_x;
if (zooming_to_initial_rect) {
// If we are zooming to the first rect, ignore any new incoming detections.
height = last_measured_height_;
offset_x = last_measured_x_offset_;
offset_y = last_measured_y_offset_;
} else if (only_required_found) {
// Convert bounds to tilt/zoom and in pixel coordinates.
MP_RETURN_IF_ERROR(ConvertToPanTiltZoom(xmin, xmax, ymin, ymax, &offset_y,
&offset_x, &height));
// A only required detection was found.
last_only_required_detection_ = cc->InputTimestamp().Microseconds();
last_measured_height_ = height;
@@ -383,7 +560,9 @@ absl::Status ContentZoomingCalculator::Process(
options_.us_before_zoomout()) {
// No only_require detections found within salient regions packets
// arriving since us_before_zoomout duration.
height = max_frame_value_ * frame_height_;
height = max_frame_value_ * frame_height_ +
(options_.kinematic_options_zoom().min_motion_to_reframe() *
(static_cast<float>(frame_height_) / kFieldOfView));
offset_x = (target_aspect_ * height) / 2;
offset_y = frame_height_ / 2;
} else {
@@ -463,17 +642,44 @@ absl::Status ContentZoomingCalculator::Process(
.AddPacket(Adopt(features.release()).At(cc->InputTimestamp()));
}
if (first_rect_timestamp_ == Timestamp::Unset() &&
options_.us_to_first_rect() != 0) {
first_rect_timestamp_ = cc->InputTimestamp();
first_rect_.set_x_center(path_offset_x / static_cast<float>(frame_width_));
first_rect_.set_width(path_height * target_aspect_ /
static_cast<float>(frame_width_));
first_rect_.set_y_center(path_offset_y / static_cast<float>(frame_height_));
first_rect_.set_height(path_height / static_cast<float>(frame_height_));
// After setting the first rectangle, check whether we should zoom to it.
zooming_to_initial_rect = IsZoomingToInitialRect(cc->InputTimestamp());
}
// Transmit downstream to glcroppingcalculator.
if (cc->Outputs().HasTag(kCropRect)) {
auto gpu_rect = absl::make_unique<mediapipe::Rect>();
gpu_rect->set_x_center(path_offset_x);
gpu_rect->set_width(path_height * target_aspect_);
gpu_rect->set_y_center(path_offset_y);
gpu_rect->set_height(path_height);
std::unique_ptr<mediapipe::Rect> gpu_rect;
if (zooming_to_initial_rect) {
auto rect = GetInitialZoomingRect(frame_width, frame_height,
cc->InputTimestamp());
MP_RETURN_IF_ERROR(rect.status());
gpu_rect = absl::make_unique<mediapipe::Rect>(*rect);
} else {
gpu_rect = absl::make_unique<mediapipe::Rect>();
gpu_rect->set_x_center(path_offset_x);
gpu_rect->set_width(path_height * target_aspect_);
gpu_rect->set_y_center(path_offset_y);
gpu_rect->set_height(path_height);
}
cc->Outputs().Tag(kCropRect).Add(gpu_rect.release(),
Timestamp(cc->InputTimestamp()));
}
if (cc->Outputs().HasTag(kFirstCropRect)) {
cc->Outputs()
.Tag(kFirstCropRect)
.Add(new mediapipe::NormalizedRect(first_rect_),
Timestamp(cc->InputTimestamp()));
}
return absl::OkStatus();
}
mediapipe/examples/desktop/autoflip/calculators/content_zooming_calculator.proto
+11 -1
View File
@@ -19,7 +19,7 @@ package mediapipe.autoflip;
import "mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver.proto";
import "mediapipe/framework/calculator.proto";
// NextTag: 14
// NextTag: 17
message ContentZoomingCalculatorOptions {
extend mediapipe.CalculatorOptions {
optional ContentZoomingCalculatorOptions ext = 313091992;
@@ -55,6 +55,16 @@ message ContentZoomingCalculatorOptions {
// Defines the smallest value in degrees the camera is permitted to zoom.
optional float max_zoom_value_deg = 13 [default = 35];
// Whether to keep state between frames or to compute the final crop rect.
optional bool is_stateless = 14 [default = false];
// Duration (in MicroSeconds) for moving to the first crop rect.
optional int64 us_to_first_rect = 15 [default = 0];
// Duration (in MicroSeconds) to delay moving to the first crop rect.
// Used only if us_to_first_rect is set and is interpreted as part of the
// us_to_first_rect time budget.
optional int64 us_to_first_rect_delay = 16 [default = 0];
// Deprecated parameters
optional KinematicOptions kinematic_options = 2 [deprecated = true];
optional int64 min_motion_to_reframe = 4 [deprecated = true];
mediapipe/examples/desktop/autoflip/calculators/content_zooming_calculator_state.h
+38
View File
@@ -0,0 +1,38 @@
#ifndef MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_CALCULATORS_CONTENT_ZOOMING_CALCULATOR_STATE_H_
#define MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_CALCULATORS_CONTENT_ZOOMING_CALCULATOR_STATE_H_
#include <optional>
#include "mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver.h"
#include "mediapipe/framework/formats/rect.pb.h"
#include "mediapipe/framework/timestamp.h"
namespace mediapipe {
namespace autoflip {
struct ContentZoomingCalculatorState {
int frame_height = -1;
int frame_width = -1;
// Path solver used to smooth top/bottom border crop values.
KinematicPathSolver path_solver_zoom;
KinematicPathSolver path_solver_pan;
KinematicPathSolver path_solver_tilt;
// Stores the time of the first crop rectangle.
Timestamp first_rect_timestamp;
// Stores the first crop rectangle.
mediapipe::NormalizedRect first_rect;
// Stores the time of the last "only_required" input.
int64 last_only_required_detection = 0;
// Rect values of last message with detection(s).
int last_measured_height = 0;
int last_measured_x_offset = 0;
int last_measured_y_offset = 0;
};
using ContentZoomingCalculatorStateCacheType =
std::optional<ContentZoomingCalculatorState>;
} // namespace autoflip
} // namespace mediapipe
#endif // MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_CALCULATORS_CONTENT_ZOOMING_CALCULATOR_STATE_H_
mediapipe/examples/desktop/autoflip/calculators/content_zooming_calculator_test.cc
+259 -14
View File
@@ -16,6 +16,7 @@
#include "mediapipe/examples/desktop/autoflip/autoflip_messages.pb.h"
#include "mediapipe/examples/desktop/autoflip/calculators/content_zooming_calculator.pb.h"
#include "mediapipe/examples/desktop/autoflip/calculators/content_zooming_calculator_state.h"
#include "mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_runner.h"
@@ -109,6 +110,7 @@ const char kConfigD[] = R"(
input_stream: "VIDEO_SIZE:size"
input_stream: "DETECTIONS:detections"
output_stream: "CROP_RECT:rect"
output_stream: "FIRST_CROP_RECT:first_rect"
options: {
[mediapipe.autoflip.ContentZoomingCalculatorOptions.ext]: {
max_zoom_value_deg: 0
@@ -147,19 +149,24 @@ void AddDetectionFrameSize(const cv::Rect_<float>& position, const int64 time,
const int width, const int height,
CalculatorRunner* runner) {
auto detections = std::make_unique<std::vector<mediapipe::Detection>>();
mediapipe::Detection detection;
detection.mutable_location_data()->set_format(
mediapipe::LocationData::RELATIVE_BOUNDING_BOX);
detection.mutable_location_data()
->mutable_relative_bounding_box()
->set_height(position.height);
detection.mutable_location_data()->mutable_relative_bounding_box()->set_width(
position.width);
detection.mutable_location_data()->mutable_relative_bounding_box()->set_xmin(
position.x);
detection.mutable_location_data()->mutable_relative_bounding_box()->set_ymin(
position.y);
detections->push_back(detection);
if (position.width > 0 && position.height > 0) {
mediapipe::Detection detection;
detection.mutable_location_data()->set_format(
mediapipe::LocationData::RELATIVE_BOUNDING_BOX);
detection.mutable_location_data()
->mutable_relative_bounding_box()
->set_height(position.height);
detection.mutable_location_data()
->mutable_relative_bounding_box()
->set_width(position.width);
detection.mutable_location_data()
->mutable_relative_bounding_box()
->set_xmin(position.x);
detection.mutable_location_data()
->mutable_relative_bounding_box()
->set_ymin(position.y);
detections->push_back(detection);
}
runner->MutableInputs()
->Tag("DETECTIONS")
.packets.push_back(Adopt(detections.release()).At(Timestamp(time)));
@@ -185,7 +192,6 @@ void CheckCropRect(const int x_center, const int y_center, const int width,
EXPECT_EQ(rect.width(), width);
EXPECT_EQ(rect.height(), height);
}
TEST(ContentZoomingCalculatorTest, ZoomTest) {
auto runner = ::absl::make_unique<CalculatorRunner>(
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfigA));
@@ -244,6 +250,46 @@ TEST(ContentZoomingCalculatorTest, PanConfig) {
runner->Outputs().Tag("CROP_RECT").packets);
}
TEST(ContentZoomingCalculatorTest, PanConfigWithCache) {
mediapipe::autoflip::ContentZoomingCalculatorStateCacheType cache;
auto config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfigD);
config.add_input_side_packet("STATE_CACHE:state_cache");
auto* options = config.mutable_options()->MutableExtension(
ContentZoomingCalculatorOptions::ext);
options->mutable_kinematic_options_pan()->set_min_motion_to_reframe(0.0);
options->mutable_kinematic_options_pan()->set_update_rate_seconds(2);
options->mutable_kinematic_options_tilt()->set_min_motion_to_reframe(50.0);
options->mutable_kinematic_options_zoom()->set_min_motion_to_reframe(50.0);
{
auto runner = ::absl::make_unique<CalculatorRunner>(config);
runner->MutableSidePackets()->Tag("STATE_CACHE") = MakePacket<
mediapipe::autoflip::ContentZoomingCalculatorStateCacheType*>(&cache);
AddDetection(cv::Rect_<float>(.4, .5, .1, .1), 0, runner.get());
MP_ASSERT_OK(runner->Run());
CheckCropRect(450, 550, 111, 111, 0,
runner->Outputs().Tag("CROP_RECT").packets);
}
{
auto runner = ::absl::make_unique<CalculatorRunner>(config);
runner->MutableSidePackets()->Tag("STATE_CACHE") = MakePacket<
mediapipe::autoflip::ContentZoomingCalculatorStateCacheType*>(&cache);
AddDetection(cv::Rect_<float>(.45, .55, .15, .15), 1000000, runner.get());
MP_ASSERT_OK(runner->Run());
CheckCropRect(483, 550, 111, 111, 0,
runner->Outputs().Tag("CROP_RECT").packets);
}
// Now repeat the last frame for a new runner without the cache to see a reset
{
auto runner = ::absl::make_unique<CalculatorRunner>(config);
runner->MutableSidePackets()->Tag("STATE_CACHE") = MakePacket<
mediapipe::autoflip::ContentZoomingCalculatorStateCacheType*>(nullptr);
AddDetection(cv::Rect_<float>(.45, .55, .15, .15), 2000000, runner.get());
MP_ASSERT_OK(runner->Run());
CheckCropRect(525, 625, 166, 166, 0, // Without a cache, state was lost.
runner->Outputs().Tag("CROP_RECT").packets);
}
}
TEST(ContentZoomingCalculatorTest, TiltConfig) {
auto config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfigD);
auto* options = config.mutable_options()->MutableExtension(
@@ -280,6 +326,46 @@ TEST(ContentZoomingCalculatorTest, ZoomConfig) {
runner->Outputs().Tag("CROP_RECT").packets);
}
TEST(ContentZoomingCalculatorTest, ZoomConfigWithCache) {
mediapipe::autoflip::ContentZoomingCalculatorStateCacheType cache;
auto config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfigD);
config.add_input_side_packet("STATE_CACHE:state_cache");
auto* options = config.mutable_options()->MutableExtension(
ContentZoomingCalculatorOptions::ext);
options->mutable_kinematic_options_pan()->set_min_motion_to_reframe(50.0);
options->mutable_kinematic_options_tilt()->set_min_motion_to_reframe(50.0);
options->mutable_kinematic_options_zoom()->set_min_motion_to_reframe(0.0);
options->mutable_kinematic_options_zoom()->set_update_rate_seconds(2);
{
auto runner = ::absl::make_unique<CalculatorRunner>(config);
runner->MutableSidePackets()->Tag("STATE_CACHE") = MakePacket<
mediapipe::autoflip::ContentZoomingCalculatorStateCacheType*>(&cache);
AddDetection(cv::Rect_<float>(.4, .5, .1, .1), 0, runner.get());
MP_ASSERT_OK(runner->Run());
CheckCropRect(450, 550, 111, 111, 0,
runner->Outputs().Tag("CROP_RECT").packets);
}
{
auto runner = ::absl::make_unique<CalculatorRunner>(config);
runner->MutableSidePackets()->Tag("STATE_CACHE") = MakePacket<
mediapipe::autoflip::ContentZoomingCalculatorStateCacheType*>(&cache);
AddDetection(cv::Rect_<float>(.45, .55, .15, .15), 1000000, runner.get());
MP_ASSERT_OK(runner->Run());
CheckCropRect(450, 550, 139, 139, 0,
runner->Outputs().Tag("CROP_RECT").packets);
}
// Now repeat the last frame for a new runner without the cache to see a reset
{
auto runner = ::absl::make_unique<CalculatorRunner>(config);
runner->MutableSidePackets()->Tag("STATE_CACHE") = MakePacket<
mediapipe::autoflip::ContentZoomingCalculatorStateCacheType*>(nullptr);
AddDetection(cv::Rect_<float>(.45, .55, .15, .15), 2000000, runner.get());
MP_ASSERT_OK(runner->Run());
CheckCropRect(525, 625, 166, 166, 0, // Without a cache, state was lost.
runner->Outputs().Tag("CROP_RECT").packets);
}
}
TEST(ContentZoomingCalculatorTest, MinAspectBorderValues) {
auto runner = ::absl::make_unique<CalculatorRunner>(
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfigB));
@@ -509,6 +595,32 @@ TEST(ContentZoomingCalculatorTest, ResolutionChangeStationary) {
runner->Outputs().Tag("CROP_RECT").packets);
}
TEST(ContentZoomingCalculatorTest, ResolutionChangeStationaryWithCache) {
mediapipe::autoflip::ContentZoomingCalculatorStateCacheType cache;
auto config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfigD);
config.add_input_side_packet("STATE_CACHE:state_cache");
{
auto runner = ::absl::make_unique<CalculatorRunner>(config);
runner->MutableSidePackets()->Tag("STATE_CACHE") = MakePacket<
mediapipe::autoflip::ContentZoomingCalculatorStateCacheType*>(&cache);
AddDetectionFrameSize(cv::Rect_<float>(.4, .4, .2, .2), 0, 1000, 1000,
runner.get());
MP_ASSERT_OK(runner->Run());
CheckCropRect(500, 500, 222, 222, 0,
runner->Outputs().Tag("CROP_RECT").packets);
}
{
auto runner = ::absl::make_unique<CalculatorRunner>(config);
runner->MutableSidePackets()->Tag("STATE_CACHE") = MakePacket<
mediapipe::autoflip::ContentZoomingCalculatorStateCacheType*>(&cache);
AddDetectionFrameSize(cv::Rect_<float>(.4, .4, .2, .2), 1, 500, 500,
runner.get());
MP_ASSERT_OK(runner->Run());
CheckCropRect(500 * 0.5, 500 * 0.5, 222 * 0.5, 222 * 0.5, 0,
runner->Outputs().Tag("CROP_RECT").packets);
}
}
TEST(ContentZoomingCalculatorTest, ResolutionChangeZooming) {
auto config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfigD);
auto runner = ::absl::make_unique<CalculatorRunner>(config);
@@ -527,6 +639,37 @@ TEST(ContentZoomingCalculatorTest, ResolutionChangeZooming) {
runner->Outputs().Tag("CROP_RECT").packets);
}
TEST(ContentZoomingCalculatorTest, ResolutionChangeZoomingWithCache) {
mediapipe::autoflip::ContentZoomingCalculatorStateCacheType cache;
auto config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfigD);
config.add_input_side_packet("STATE_CACHE:state_cache");
{
auto runner = ::absl::make_unique<CalculatorRunner>(config);
runner->MutableSidePackets()->Tag("STATE_CACHE") = MakePacket<
mediapipe::autoflip::ContentZoomingCalculatorStateCacheType*>(&cache);
AddDetectionFrameSize(cv::Rect_<float>(.1, .1, .8, .8), 0, 1000, 1000,
runner.get());
MP_ASSERT_OK(runner->Run());
CheckCropRect(500, 500, 888, 888, 0,
runner->Outputs().Tag("CROP_RECT").packets);
}
// The second runner should just resume based on state from the first runner.
{
auto runner = ::absl::make_unique<CalculatorRunner>(config);
runner->MutableSidePackets()->Tag("STATE_CACHE") = MakePacket<
mediapipe::autoflip::ContentZoomingCalculatorStateCacheType*>(&cache);
AddDetectionFrameSize(cv::Rect_<float>(.4, .4, .2, .2), 1000000, 1000, 1000,
runner.get());
AddDetectionFrameSize(cv::Rect_<float>(.4, .4, .2, .2), 2000000, 500, 500,
runner.get());
MP_ASSERT_OK(runner->Run());
CheckCropRect(500, 500, 588, 588, 0,
runner->Outputs().Tag("CROP_RECT").packets);
CheckCropRect(500 * 0.5, 500 * 0.5, 288 * 0.5, 288 * 0.5, 1,
runner->Outputs().Tag("CROP_RECT").packets);
}
}
TEST(ContentZoomingCalculatorTest, MaxZoomValue) {
auto config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfigD);
auto* options = config.mutable_options()->MutableExtension(
@@ -540,6 +683,108 @@ TEST(ContentZoomingCalculatorTest, MaxZoomValue) {
CheckCropRect(500, 500, 916, 916, 0,
runner->Outputs().Tag("CROP_RECT").packets);
}
TEST(ContentZoomingCalculatorTest, MaxZoomOutValue) {
auto config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfigD);
auto* options = config.mutable_options()->MutableExtension(
ContentZoomingCalculatorOptions::ext);
options->set_scale_factor(1.0);
options->mutable_kinematic_options_zoom()->set_min_motion_to_reframe(5.0);
auto runner = ::absl::make_unique<CalculatorRunner>(config);
AddDetectionFrameSize(cv::Rect_<float>(.025, .025, .95, .95), 0, 1000, 1000,
runner.get());
AddDetectionFrameSize(cv::Rect_<float>(0, 0, -1, -1), 1000000, 1000, 1000,
runner.get());
AddDetectionFrameSize(cv::Rect_<float>(0, 0, -1, -1), 2000000, 1000, 1000,
runner.get());
MP_ASSERT_OK(runner->Run());
// 55/60 * 1000 = 916
CheckCropRect(500, 500, 950, 950, 0,
runner->Outputs().Tag("CROP_RECT").packets);
CheckCropRect(500, 500, 1000, 1000, 2,
runner->Outputs().Tag("CROP_RECT").packets);
}
TEST(ContentZoomingCalculatorTest, StartZoomedOut) {
auto config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfigD);
auto* options = config.mutable_options()->MutableExtension(
ContentZoomingCalculatorOptions::ext);
options->set_us_to_first_rect(1000000);
options->set_us_to_first_rect_delay(500000);
auto runner = ::absl::make_unique<CalculatorRunner>(config);
AddDetectionFrameSize(cv::Rect_<float>(.4, .4, .2, .2), 0, 1000, 1000,
runner.get());
AddDetectionFrameSize(cv::Rect_<float>(.4, .4, .2, .2), 400000, 1000, 1000,
runner.get());
AddDetectionFrameSize(cv::Rect_<float>(.4, .4, .2, .2), 800000, 1000, 1000,
runner.get());
AddDetectionFrameSize(cv::Rect_<float>(.4, .4, .2, .2), 1000000, 1000, 1000,
runner.get());
AddDetectionFrameSize(cv::Rect_<float>(.4, .4, .2, .2), 1500000, 1000, 1000,
runner.get());
MP_ASSERT_OK(runner->Run());
CheckCropRect(500, 500, 1000, 1000, 0,
runner->Outputs().Tag("CROP_RECT").packets);
CheckCropRect(500, 500, 1000, 1000, 1,
runner->Outputs().Tag("CROP_RECT").packets);
CheckCropRect(500, 500, 470, 470, 2,
runner->Outputs().Tag("CROP_RECT").packets);
CheckCropRect(500, 500, 222, 222, 3,
runner->Outputs().Tag("CROP_RECT").packets);
CheckCropRect(500, 500, 222, 222, 4,
runner->Outputs().Tag("CROP_RECT").packets);
}
TEST(ContentZoomingCalculatorTest, ProvidesZeroSizeFirstRectWithoutDetections) {
auto config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfigD);
auto runner = ::absl::make_unique<CalculatorRunner>(config);
auto input_size = ::absl::make_unique<std::pair<int, int>>(1000, 1000);
runner->MutableInputs()
->Tag("VIDEO_SIZE")
.packets.push_back(Adopt(input_size.release()).At(Timestamp(0)));
MP_ASSERT_OK(runner->Run());
const std::vector<Packet>& output_packets =
runner->Outputs().Tag("FIRST_CROP_RECT").packets;
ASSERT_EQ(output_packets.size(), 1);
const auto& rect = output_packets[0].Get<mediapipe::NormalizedRect>();
EXPECT_EQ(rect.x_center(), 0);
EXPECT_EQ(rect.y_center(), 0);
EXPECT_EQ(rect.width(), 0);
EXPECT_EQ(rect.height(), 0);
}
TEST(ContentZoomingCalculatorTest, ProvidesConstantFirstRect) {
auto config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfigD);
auto* options = config.mutable_options()->MutableExtension(
ContentZoomingCalculatorOptions::ext);
options->set_us_to_first_rect(500000);
auto runner = ::absl::make_unique<CalculatorRunner>(config);
AddDetectionFrameSize(cv::Rect_<float>(.4, .4, .2, .2), 0, 1000, 1000,
runner.get());
AddDetectionFrameSize(cv::Rect_<float>(.4, .4, .2, .2), 500000, 1000, 1000,
runner.get());
AddDetectionFrameSize(cv::Rect_<float>(.4, .4, .2, .2), 1000000, 1000, 1000,
runner.get());
AddDetectionFrameSize(cv::Rect_<float>(.4, .4, .2, .2), 1500000, 1000, 1000,
runner.get());
MP_ASSERT_OK(runner->Run());
const std::vector<Packet>& output_packets =
runner->Outputs().Tag("FIRST_CROP_RECT").packets;
ASSERT_EQ(output_packets.size(), 4);
const auto& first_rect = output_packets[0].Get<mediapipe::NormalizedRect>();
EXPECT_NEAR(first_rect.x_center(), 0.5, 0.05);
EXPECT_NEAR(first_rect.y_center(), 0.5, 0.05);
EXPECT_NEAR(first_rect.width(), 0.222, 0.05);
EXPECT_NEAR(first_rect.height(), 0.222, 0.05);
for (int i = 1; i < 4; ++i) {
const auto& rect = output_packets[i].Get<mediapipe::NormalizedRect>();
EXPECT_EQ(first_rect.x_center(), rect.x_center());
EXPECT_EQ(first_rect.y_center(), rect.y_center());
EXPECT_EQ(first_rect.width(), rect.width());
EXPECT_EQ(first_rect.height(), rect.height());
}
}
} // namespace
} // namespace autoflip
mediapipe/examples/desktop/autoflip/calculators/shot_boundary_calculator_test.cc
+1 -1
View File
@@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include "absl/flags/flag.h"
#include "absl/strings/string_view.h"
#include "mediapipe/examples/desktop/autoflip/calculators/shot_boundary_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
@@ -19,7 +20,6 @@
#include "mediapipe/framework/deps/file_path.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/formats/image_frame_opencv.h"
#include "mediapipe/framework/port/commandlineflags.h"
#include "mediapipe/framework/port/gmock.h"
#include "mediapipe/framework/port/gtest.h"
#include "mediapipe/framework/port/opencv_core_inc.h"
mediapipe/examples/desktop/autoflip/quality/BUILD
+6 -4
View File
@@ -46,7 +46,9 @@ proto_library(
mediapipe_cc_proto_library(
name = "kinematic_path_solver_cc_proto",
srcs = ["kinematic_path_solver.proto"],
visibility = ["//mediapipe/examples:__subpackages__"],
visibility = [
"//mediapipe/examples:__subpackages__",
],
deps = [":kinematic_path_solver_proto"],
)
@@ -96,11 +98,11 @@ cc_library(
deps = [
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/formats:image_frame_opencv",
"//mediapipe/framework/port:commandlineflags",
"//mediapipe/framework/port:opencv_core",
"//mediapipe/framework/port:opencv_imgproc",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"@com_google_absl//absl/flags:flag",
],
)
@@ -249,10 +251,10 @@ cc_test(
":scene_camera_motion_analyzer",
"//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto",
"//mediapipe/framework/deps:file_path",
"//mediapipe/framework/port:commandlineflags",
"//mediapipe/framework/port:file_helpers",
"//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:status",
"@com_google_absl//absl/flags:flag",
"@com_google_absl//absl/strings",
],
)
@@ -280,13 +282,13 @@ cc_test(
"//mediapipe/framework/deps:file_path",
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/formats:image_frame_opencv",
"//mediapipe/framework/port:commandlineflags",
"//mediapipe/framework/port:file_helpers",
"//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:opencv_imgcodecs",
"//mediapipe/framework/port:opencv_imgproc",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"@com_google_absl//absl/flags:flag",
"@com_google_absl//absl/strings",
],
)
mediapipe/examples/desktop/autoflip/quality/padding_effect_generator_test.cc
+4 -3
View File
@@ -14,11 +14,11 @@
#include "mediapipe/examples/desktop/autoflip/quality/padding_effect_generator.h"
#include "absl/flags/flag.h"
#include "absl/strings/str_cat.h"
#include "mediapipe/framework/deps/file_path.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/formats/image_frame_opencv.h"
#include "mediapipe/framework/port/commandlineflags.h"
#include "mediapipe/framework/port/file_helpers.h"
#include "mediapipe/framework/port/gmock.h"
#include "mediapipe/framework/port/gtest.h"
@@ -28,8 +28,9 @@
#include "mediapipe/framework/port/status_builder.h"
#include "mediapipe/framework/port/status_matchers.h"
DEFINE_string(input_image, "", "The path to an input image.");
DEFINE_string(output_folder, "", "The folder to output test result images.");
ABSL_FLAG(std::string, input_image, "", "The path to an input image.");
ABSL_FLAG(std::string, output_folder, "",
"The folder to output test result images.");
namespace mediapipe {
namespace autoflip {
mediapipe/examples/desktop/autoflip/quality/scene_camera_motion_analyzer_test.cc
+1 -1
View File
@@ -19,12 +19,12 @@
#include <string>
#include <vector>
#include "absl/flags/flag.h"
#include "absl/strings/str_split.h"
#include "mediapipe/examples/desktop/autoflip/autoflip_messages.pb.h"
#include "mediapipe/examples/desktop/autoflip/quality/focus_point.pb.h"
#include "mediapipe/examples/desktop/autoflip/quality/piecewise_linear_function.h"
#include "mediapipe/framework/deps/file_path.h"
#include "mediapipe/framework/port/commandlineflags.h"
#include "mediapipe/framework/port/file_helpers.h"
#include "mediapipe/framework/port/gmock.h"
#include "mediapipe/framework/port/gtest.h"
mediapipe/examples/desktop/autoflip/quality/scene_cropping_viz.cc
+1 -1
View File
@@ -202,7 +202,7 @@ absl::Status DrawFocusPointAndCropWindow(
const auto& point = focus_point_frames[i].point(j);
const int x = point.norm_point_x() * scene_frame.cols;
const int y = point.norm_point_y() * scene_frame.rows;
cv::circle(viz_mat, cv::Point(x, y), 3, kRed, CV_FILLED);
cv::circle(viz_mat, cv::Point(x, y), 3, kRed, cv::FILLED);
center_x += x;
center_y += y;
}
mediapipe/examples/desktop/demo_run_graph_main.cc
+11 -11
View File
@@ -15,10 +15,11 @@
// An example of sending OpenCV webcam frames into a MediaPipe graph.
#include <cstdlib>
#include "absl/flags/flag.h"
#include "absl/flags/parse.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/formats/image_frame_opencv.h"
#include "mediapipe/framework/port/commandlineflags.h"
#include "mediapipe/framework/port/file_helpers.h"
#include "mediapipe/framework/port/opencv_highgui_inc.h"
#include "mediapipe/framework/port/opencv_imgproc_inc.h"
@@ -30,15 +31,14 @@ constexpr char kInputStream[] = "input_video";
constexpr char kOutputStream[] = "output_video";
constexpr char kWindowName[] = "MediaPipe";
DEFINE_string(
calculator_graph_config_file, "",
"Name of file containing text format CalculatorGraphConfig proto.");
DEFINE_string(input_video_path, "",
"Full path of video to load. "
"If not provided, attempt to use a webcam.");
DEFINE_string(output_video_path, "",
"Full path of where to save result (.mp4 only). "
"If not provided, show result in a window.");
ABSL_FLAG(std::string, calculator_graph_config_file, "",
"Name of file containing text format CalculatorGraphConfig proto.");
ABSL_FLAG(std::string, input_video_path, "",
"Full path of video to load. "
"If not provided, attempt to use a webcam.");
ABSL_FLAG(std::string, output_video_path, "",
"Full path of where to save result (.mp4 only). "
"If not provided, show result in a window.");
absl::Status RunMPPGraph() {
std::string calculator_graph_config_contents;
@@ -148,7 +148,7 @@ absl::Status RunMPPGraph() {
int main(int argc, char** argv) {
google::InitGoogleLogging(argv[0]);
gflags::ParseCommandLineFlags(&argc, &argv, true);
absl::ParseCommandLine(argc, argv);
absl::Status run_status = RunMPPGraph();
if (!run_status.ok()) {
LOG(ERROR) << "Failed to run the graph: " << run_status.message();
mediapipe/examples/desktop/demo_run_graph_main_gpu.cc
+11 -11
View File
@@ -16,10 +16,11 @@
// This example requires a linux computer and a GPU with EGL support drivers.
#include <cstdlib>
#include "absl/flags/flag.h"
#include "absl/flags/parse.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/formats/image_frame_opencv.h"
#include "mediapipe/framework/port/commandlineflags.h"
#include "mediapipe/framework/port/file_helpers.h"
#include "mediapipe/framework/port/opencv_highgui_inc.h"
#include "mediapipe/framework/port/opencv_imgproc_inc.h"
@@ -34,15 +35,14 @@ constexpr char kInputStream[] = "input_video";
constexpr char kOutputStream[] = "output_video";
constexpr char kWindowName[] = "MediaPipe";
DEFINE_string(
calculator_graph_config_file, "",
"Name of file containing text format CalculatorGraphConfig proto.");
DEFINE_string(input_video_path, "",
"Full path of video to load. "
"If not provided, attempt to use a webcam.");
DEFINE_string(output_video_path, "",
"Full path of where to save result (.mp4 only). "
"If not provided, show result in a window.");
ABSL_FLAG(std::string, calculator_graph_config_file, "",
"Name of file containing text format CalculatorGraphConfig proto.");
ABSL_FLAG(std::string, input_video_path, "",
"Full path of video to load. "
"If not provided, attempt to use a webcam.");
ABSL_FLAG(std::string, output_video_path, "",
"Full path of where to save result (.mp4 only). "
"If not provided, show result in a window.");
absl::Status RunMPPGraph() {
std::string calculator_graph_config_contents;
@@ -191,7 +191,7 @@ absl::Status RunMPPGraph() {
int main(int argc, char** argv) {
google::InitGoogleLogging(argv[0]);
gflags::ParseCommandLineFlags(&argc, &argv, true);
absl::ParseCommandLine(argc, argv);
absl::Status run_status = RunMPPGraph();
if (!run_status.ok()) {
LOG(ERROR) << "Failed to run the graph: " << run_status.message();
mediapipe/examples/desktop/iris_tracking/BUILD
+2 -1
View File
@@ -23,7 +23,6 @@ cc_binary(
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/formats:image_frame_opencv",
"//mediapipe/framework/port:commandlineflags",
"//mediapipe/framework/port:file_helpers",
"//mediapipe/framework/port:opencv_highgui",
"//mediapipe/framework/port:opencv_imgproc",
@@ -31,6 +30,8 @@ cc_binary(
"//mediapipe/framework/port:parse_text_proto",
"//mediapipe/framework/port:status",
"//mediapipe/graphs/iris_tracking:iris_depth_cpu_deps",
"@com_google_absl//absl/flags:flag",
"@com_google_absl//absl/flags:parse",
],
)
mediapipe/examples/desktop/iris_tracking/iris_depth_from_image_desktop.cc
+9 -8
View File
@@ -17,11 +17,12 @@
#include <cstdlib>
#include <memory>
#include "absl/flags/flag.h"
#include "absl/flags/parse.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/formats/image_frame_opencv.h"
#include "mediapipe/framework/port/canonical_errors.h"
#include "mediapipe/framework/port/commandlineflags.h"
#include "mediapipe/framework/port/file_helpers.h"
#include "mediapipe/framework/port/opencv_highgui_inc.h"
#include "mediapipe/framework/port/opencv_imgproc_inc.h"
@@ -38,12 +39,12 @@ constexpr char kCalculatorGraphConfigFile[] =
"mediapipe/graphs/iris_tracking/iris_depth_cpu.pbtxt";
constexpr float kMicrosPerSecond = 1e6;
DEFINE_string(input_image_path, "",
"Full path of image to load. "
"If not provided, nothing will run.");
DEFINE_string(output_image_path, "",
"Full path of where to save image result (.jpg only). "
"If not provided, show result in a window.");
ABSL_FLAG(std::string, input_image_path, "",
"Full path of image to load. "
"If not provided, nothing will run.");
ABSL_FLAG(std::string, output_image_path, "",
"Full path of where to save image result (.jpg only). "
"If not provided, show result in a window.");
namespace {
@@ -148,7 +149,7 @@ absl::Status RunMPPGraph() {
int main(int argc, char** argv) {
google::InitGoogleLogging(argv[0]);
gflags::ParseCommandLineFlags(&argc, &argv, true);
absl::ParseCommandLine(argc, argv);
absl::Status run_status = RunMPPGraph();
if (!run_status.ok()) {
LOG(ERROR) << "Failed to run the graph: " << run_status.message();
mediapipe/examples/desktop/media_sequence/BUILD
+2 -1
View File
@@ -21,11 +21,12 @@ cc_library(
srcs = ["run_graph_file_io_main.cc"],
deps = [
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/port:commandlineflags",
"//mediapipe/framework/port:file_helpers",
"//mediapipe/framework/port:map_util",
"//mediapipe/framework/port:parse_text_proto",
"//mediapipe/framework/port:status",
"@com_google_absl//absl/flags:flag",
"@com_google_absl//absl/flags:parse",
"@com_google_absl//absl/strings",
],
)
mediapipe/examples/desktop/media_sequence/run_graph_file_io_main.cc
+14 -14
View File
@@ -17,26 +17,26 @@
// to disk.
#include <cstdlib>
#include "absl/flags/flag.h"
#include "absl/flags/parse.h"
#include "absl/strings/str_split.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/port/commandlineflags.h"
#include "mediapipe/framework/port/file_helpers.h"
#include "mediapipe/framework/port/map_util.h"
#include "mediapipe/framework/port/parse_text_proto.h"
#include "mediapipe/framework/port/status.h"
DEFINE_string(
calculator_graph_config_file, "",
"Name of file containing text format CalculatorGraphConfig proto.");
DEFINE_string(input_side_packets, "",
"Comma-separated list of key=value pairs specifying side packets "
"and corresponding file paths for the CalculatorGraph. The side "
"packets are read from the files and fed to the graph as strings "
"even if they represent doubles, floats, etc.");
DEFINE_string(output_side_packets, "",
"Comma-separated list of key=value pairs specifying the output "
"side packets and paths to write to disk for the "
"CalculatorGraph.");
ABSL_FLAG(std::string, calculator_graph_config_file, "",
"Name of file containing text format CalculatorGraphConfig proto.");
ABSL_FLAG(std::string, input_side_packets, "",
"Comma-separated list of key=value pairs specifying side packets "
"and corresponding file paths for the CalculatorGraph. The side "
"packets are read from the files and fed to the graph as strings "
"even if they represent doubles, floats, etc.");
ABSL_FLAG(std::string, output_side_packets, "",
"Comma-separated list of key=value pairs specifying the output "
"side packets and paths to write to disk for the "
"CalculatorGraph.");
absl::Status RunMPPGraph() {
std::string calculator_graph_config_contents;
@@ -85,7 +85,7 @@ absl::Status RunMPPGraph() {
int main(int argc, char** argv) {
google::InitGoogleLogging(argv[0]);
gflags::ParseCommandLineFlags(&argc, &argv, true);
absl::ParseCommandLine(argc, argv);
absl::Status run_status = RunMPPGraph();
if (!run_status.ok()) {
LOG(ERROR) << "Failed to run the graph: " << run_status.message();
mediapipe/examples/desktop/object_detection_3d/BUILD
+1 -1
View File
@@ -20,7 +20,7 @@ package(default_visibility = ["//mediapipe/examples:__subpackages__"])
# To run 3D object detection for shoes,
# bazel-bin/mediapipe/examples/desktop/object_detection_3d/objectron_cpu \
# --calculator_graph_config_file=mediapipe/graphs/object_detection_3d/objectron_desktop_cpu.pbtxt \
# --input_side_packets="input_video_path=<input_video_path>,box_landmark_model_path=mediapipe/models/object_detection_3d_sneakers.tflite,output_video_path=<output_video_path>,allowed_labels=Footwear"
# --input_side_packets="input_video_path=<input_video_path>,box_landmark_model_path=mediapipe/modules/objectron/object_detection_3d_sneakers.tflite,output_video_path=<output_video_path>,allowed_labels=Footwear"
# To detect objects from other categories, change box_landmark_model_path and allowed_labels accordingly.
# Chair: box_landmark_model_path=mediapipe/modules/objectron/object_detection_3d_chair.tflite,allowed_labels=Chair
# Camera: box_landmark_model_path=mediapipe/modules/objectron/object_detection_3d_camera.tflite,allowed_labels=Camera
mediapipe/examples/desktop/simple_run_graph_main.cc
+22 -22
View File
@@ -20,11 +20,12 @@
#include <string>
#include <vector>
#include "absl/flags/flag.h"
#include "absl/flags/parse.h"
#include "absl/strings/str_cat.h"
#include "absl/strings/str_split.h"
#include "absl/strings/string_view.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/port/commandlineflags.h"
#include "mediapipe/framework/port/file_helpers.h"
#include "mediapipe/framework/port/map_util.h"
#include "mediapipe/framework/port/parse_text_proto.h"
@@ -32,31 +33,30 @@
#include "mediapipe/framework/port/status.h"
#include "mediapipe/framework/port/statusor.h"
DEFINE_string(
calculator_graph_config_file, "",
"Name of file containing text format CalculatorGraphConfig proto.");
ABSL_FLAG(std::string, calculator_graph_config_file, "",
"Name of file containing text format CalculatorGraphConfig proto.");
DEFINE_string(input_side_packets, "",
"Comma-separated list of key=value pairs specifying side packets "
"for the CalculatorGraph. All values will be treated as the "
"string type even if they represent doubles, floats, etc.");
ABSL_FLAG(std::string, input_side_packets, "",
"Comma-separated list of key=value pairs specifying side packets "
"for the CalculatorGraph. All values will be treated as the "
"string type even if they represent doubles, floats, etc.");
// Local file output flags.
// Output stream
DEFINE_string(output_stream, "",
"The output stream to output to the local file in csv format.");
DEFINE_string(output_stream_file, "",
"The name of the local file to output all packets sent to "
"the stream specified with --output_stream. ");
DEFINE_bool(strip_timestamps, false,
"If true, only the packet contents (without timestamps) will be "
"written into the local file.");
ABSL_FLAG(std::string, output_stream, "",
"The output stream to output to the local file in csv format.");
ABSL_FLAG(std::string, output_stream_file, "",
"The name of the local file to output all packets sent to "
"the stream specified with --output_stream. ");
ABSL_FLAG(bool, strip_timestamps, false,
"If true, only the packet contents (without timestamps) will be "
"written into the local file.");
// Output side packets
DEFINE_string(output_side_packets, "",
"A CSV of output side packets to output to local file.");
DEFINE_string(output_side_packets_file, "",
"The name of the local file to output all side packets specified "
"with --output_side_packets. ");
ABSL_FLAG(std::string, output_side_packets, "",
"A CSV of output side packets to output to local file.");
ABSL_FLAG(std::string, output_side_packets_file, "",
"The name of the local file to output all side packets specified "
"with --output_side_packets. ");
absl::Status OutputStreamToLocalFile(mediapipe::OutputStreamPoller& poller) {
std::ofstream file;
@@ -143,7 +143,7 @@ absl::Status RunMPPGraph() {
int main(int argc, char** argv) {
google::InitGoogleLogging(argv[0]);
gflags::ParseCommandLineFlags(&argc, &argv, true);
absl::ParseCommandLine(argc, argv);
absl::Status run_status = RunMPPGraph();
if (!run_status.ok()) {
LOG(ERROR) << "Failed to run the graph: " << run_status.message();
mediapipe/examples/desktop/youtube8m/BUILD
+2 -1
View File
@@ -18,10 +18,11 @@ cc_binary(
name = "extract_yt8m_features",
srcs = ["extract_yt8m_features.cc"],
deps = [
"@com_google_absl//absl/flags:flag",
"@com_google_absl//absl/flags:parse",
"@com_google_absl//absl/strings",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/formats:matrix",
"//mediapipe/framework/port:commandlineflags",
"//mediapipe/framework/port:file_helpers",
"//mediapipe/framework/port:map_util",
"//mediapipe/framework/port:parse_text_proto",
mediapipe/examples/desktop/youtube8m/extract_yt8m_features.cc
+14 -14
View File
@@ -17,27 +17,27 @@
// to disk.
#include <cstdlib>
#include "absl/flags/flag.h"
#include "absl/flags/parse.h"
#include "absl/strings/str_split.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/matrix.h"
#include "mediapipe/framework/port/commandlineflags.h"
#include "mediapipe/framework/port/file_helpers.h"
#include "mediapipe/framework/port/map_util.h"
#include "mediapipe/framework/port/parse_text_proto.h"
#include "mediapipe/framework/port/status.h"
DEFINE_string(
calculator_graph_config_file, "",
"Name of file containing text format CalculatorGraphConfig proto.");
DEFINE_string(input_side_packets, "",
"Comma-separated list of key=value pairs specifying side packets "
"and corresponding file paths for the CalculatorGraph. The side "
"packets are read from the files and fed to the graph as strings "
"even if they represent doubles, floats, etc.");
DEFINE_string(output_side_packets, "",
"Comma-separated list of key=value pairs specifying the output "
"side packets and paths to write to disk for the "
"CalculatorGraph.");
ABSL_FLAG(std::string, calculator_graph_config_file, "",
"Name of file containing text format CalculatorGraphConfig proto.");
ABSL_FLAG(std::string, input_side_packets, "",
"Comma-separated list of key=value pairs specifying side packets "
"and corresponding file paths for the CalculatorGraph. The side "
"packets are read from the files and fed to the graph as strings "
"even if they represent doubles, floats, etc.");
ABSL_FLAG(std::string, output_side_packets, "",
"Comma-separated list of key=value pairs specifying the output "
"side packets and paths to write to disk for the "
"CalculatorGraph.");
absl::Status RunMPPGraph() {
std::string calculator_graph_config_contents;
@@ -126,7 +126,7 @@ absl::Status RunMPPGraph() {
int main(int argc, char** argv) {
google::InitGoogleLogging(argv[0]);
gflags::ParseCommandLineFlags(&argc, &argv, true);
absl::ParseCommandLine(argc, argv);
absl::Status run_status = RunMPPGraph();
if (!run_status.ok()) {
LOG(ERROR) << "Failed to run the graph: " << run_status.message();