talksik/mediapipe
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Project import generated by Copybara.

Browse Source 
GitOrigin-RevId: d073f8e21be2fcc0e503cb97c6695078b6b75310
This commit is contained in:
MediaPipe Team
2021-02-27 03:30:05 -05:00
committed by chuoling
parent 39309bedba
commit 350fbb2100
755 changed files with 16391 additions and 11075 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mediapipe/examples/android/src/java/com/google/mediapipe/apps/faceeffect/BUILD
+4 -1
View File
@@ -36,12 +36,15 @@ android_binary(
name = "faceeffect",
srcs = glob(["*.java"]),
assets = [
"//mediapipe/graphs/face_effect/data:axis.binarypb",
"//mediapipe/graphs/face_effect/data:axis.pngblob",
"//mediapipe/graphs/face_effect/data:facepaint.pngblob",
"//mediapipe/graphs/face_effect/data:glasses.binarypb",
"//mediapipe/graphs/face_effect/data:glasses.pngblob",
"//mediapipe/graphs/face_effect:face_effect_gpu.binarypb",
"//mediapipe/modules/face_detection:face_detection_front.tflite",
"//mediapipe/modules/face_geometry/data:geometry_pipeline_metadata.binarypb",
"//mediapipe/modules/face_geometry/data:geometry_pipeline_metadata_detection.binarypb",
"//mediapipe/modules/face_geometry/data:geometry_pipeline_metadata_landmarks.binarypb",
"//mediapipe/modules/face_landmark:face_landmark.tflite",
],
assets_dir = "",
mediapipe/examples/android/src/java/com/google/mediapipe/apps/faceeffect/MainActivity.java
+68 -28
View File
@@ -29,23 +29,31 @@ import com.google.mediapipe.framework.Packet;
import com.google.mediapipe.framework.PacketGetter;
import com.google.mediapipe.modules.facegeometry.FaceGeometryProto.FaceGeometry;
import com.google.mediapipe.formats.proto.MatrixDataProto.MatrixData;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/** Main activity of MediaPipe face mesh app. */
public class MainActivity extends com.google.mediapipe.apps.basic.MainActivity {
private static final String TAG = "MainActivity";
// Stream names.
private static final String IS_FACEPAINT_EFFECT_SELECTED_INPUT_STREAM_NAME =
"is_facepaint_effect_selected";
// Side packet / stream names.
private static final String USE_FACE_DETECTION_INPUT_SOURCE_INPUT_SIDE_PACKET_NAME =
"use_face_detection_input_source";
private static final String SELECTED_EFFECT_ID_INPUT_STREAM_NAME = "selected_effect_id";
private static final String OUTPUT_FACE_GEOMETRY_STREAM_NAME = "multi_face_geometry";
private static final String EFFECT_SWITCHING_HINT_TEXT = "Tap to switch between effects!";
private static final boolean USE_FACE_DETECTION_INPUT_SOURCE = false;
private static final int MATRIX_TRANSLATION_Z_INDEX = 14;
private final Object isFacepaintEffectSelectedLock = new Object();
private boolean isFacepaintEffectSelected;
private static final int SELECTED_EFFECT_ID_AXIS = 0;
private static final int SELECTED_EFFECT_ID_FACEPAINT = 1;
private static final int SELECTED_EFFECT_ID_GLASSES = 2;
private final Object effectSelectionLock = new Object();
private int selectedEffectId;
private View effectSwitchingHintView;
private GestureDetector tapGestureDetector;
@@ -60,8 +68,20 @@ public class MainActivity extends com.google.mediapipe.apps.basic.MainActivity {
ViewGroup viewGroup = findViewById(R.id.preview_display_layout);
viewGroup.addView(effectSwitchingHintView);
// By default, render the glasses effect.
isFacepaintEffectSelected = false;
// By default, render the axis effect for the face detection input source and the glasses effect
// for the face landmark input source.
if (USE_FACE_DETECTION_INPUT_SOURCE) {
selectedEffectId = SELECTED_EFFECT_ID_AXIS;
} else {
selectedEffectId = SELECTED_EFFECT_ID_GLASSES;
}
// Pass the USE_FACE_DETECTION_INPUT_SOURCE flag value as an input side packet into the graph.
Map<String, Packet> inputSidePackets = new HashMap<>();
inputSidePackets.put(
USE_FACE_DETECTION_INPUT_SOURCE_INPUT_SIDE_PACKET_NAME,
processor.getPacketCreator().createBool(USE_FACE_DETECTION_INPUT_SOURCE));
processor.setInputSidePackets(inputSidePackets);
// This callback demonstrates how the output face geometry packet can be obtained and used
// in an Android app. As an example, the Z-translation component of the face pose transform
@@ -71,12 +91,9 @@ public class MainActivity extends com.google.mediapipe.apps.basic.MainActivity {
OUTPUT_FACE_GEOMETRY_STREAM_NAME,
(packet) -> {
effectSwitchingHintView.post(
new Runnable() {
@Override
public void run() {
effectSwitchingHintView.setVisibility(View.VISIBLE);
}
});
() ->
effectSwitchingHintView.setVisibility(
USE_FACE_DETECTION_INPUT_SOURCE ? View.INVISIBLE : View.VISIBLE));
Log.d(TAG, "Received a multi face geometry packet.");
List<FaceGeometry> multiFaceGeometry =
@@ -103,30 +120,26 @@ public class MainActivity extends com.google.mediapipe.apps.basic.MainActivity {
+ "]");
});
// Alongside the input camera frame, we also send the `is_facepaint_effect_selected` boolean
// packet to indicate which effect should be rendered on this frame.
// Alongside the input camera frame, we also send the `selected_effect_id` int32 packet to
// indicate which effect should be rendered on this frame.
processor.setOnWillAddFrameListener(
(timestamp) -> {
Packet isFacepaintEffectSelectedPacket = null;
Packet selectedEffectIdPacket = null;
try {
synchronized (isFacepaintEffectSelectedLock) {
isFacepaintEffectSelectedPacket =
processor.getPacketCreator().createBool(isFacepaintEffectSelected);
synchronized (effectSelectionLock) {
selectedEffectIdPacket = processor.getPacketCreator().createInt32(selectedEffectId);
}
processor
.getGraph()
.addPacketToInputStream(
IS_FACEPAINT_EFFECT_SELECTED_INPUT_STREAM_NAME,
isFacepaintEffectSelectedPacket,
timestamp);
SELECTED_EFFECT_ID_INPUT_STREAM_NAME, selectedEffectIdPacket, timestamp);
} catch (RuntimeException e) {
Log.e(
TAG,
"Exception while adding packet to input stream while switching effects: " + e);
TAG, "Exception while adding packet to input stream while switching effects: " + e);
} finally {
if (isFacepaintEffectSelectedPacket != null) {
isFacepaintEffectSelectedPacket.release();
if (selectedEffectIdPacket != null) {
selectedEffectIdPacket.release();
}
}
});
@@ -149,8 +162,35 @@ public class MainActivity extends com.google.mediapipe.apps.basic.MainActivity {
}
private void switchEffect() {
synchronized (isFacepaintEffectSelectedLock) {
isFacepaintEffectSelected = !isFacepaintEffectSelected;
// Avoid switching the Axis effect for the face detection input source.
if (USE_FACE_DETECTION_INPUT_SOURCE) {
return;
}
// Looped effect order: glasses -> facepaint -> axis -> glasses -> ...
synchronized (effectSelectionLock) {
switch (selectedEffectId) {
case SELECTED_EFFECT_ID_AXIS:
{
selectedEffectId = SELECTED_EFFECT_ID_GLASSES;
break;
}
case SELECTED_EFFECT_ID_FACEPAINT:
{
selectedEffectId = SELECTED_EFFECT_ID_AXIS;
break;
}
case SELECTED_EFFECT_ID_GLASSES:
{
selectedEffectId = SELECTED_EFFECT_ID_FACEPAINT;
break;
}
default:
break;
}
}
}
});
mediapipe/examples/android/src/java/com/google/mediapipe/apps/iristrackinggpu/BUILD
+1
View File
@@ -60,5 +60,6 @@ android_binary(
"//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:basic_lib",
"//mediapipe/framework/formats:landmark_java_proto_lite",
"//mediapipe/java/com/google/mediapipe/framework:android_framework",
"@com_google_protobuf//:protobuf_javalite",
],
)
mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/BUILD
+2 -2
View File
@@ -90,7 +90,7 @@ genrule(
cmd = "cp $< $@",
)
MODELS_DIR = "//mediapipe/models"
MODELS_DIR = "//mediapipe/modules/objectron"
genrule(
name = "model",
@@ -165,7 +165,7 @@ android_binary(
":mesh",
":texture",
MODELS_DIR + ":object_detection_ssd_mobilenetv2_oidv4_fp16.tflite",
MODELS_DIR + ":object_detection_oidv4_labelmap.pbtxt",
MODELS_DIR + ":object_detection_oidv4_labelmap.txt",
ASSETS_DIR + ":box.obj.uuu",
ASSETS_DIR + ":classic_colors.png",
],
mediapipe/examples/android/src/java/com/google/mediapipe/apps/posetrackinggpu/BUILD
+1
View File
@@ -59,5 +59,6 @@ android_binary(
"//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:basic_lib",
"//mediapipe/framework/formats:landmark_java_proto_lite",
"//mediapipe/java/com/google/mediapipe/framework:android_framework",
"@com_google_protobuf//:protobuf_javalite",
],
)
mediapipe/examples/android/src/java/com/google/mediapipe/apps/upperbodyposetrackinggpu/BUILD
+1
View File
@@ -59,5 +59,6 @@ android_binary(
"//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:basic_lib",
"//mediapipe/framework/formats:landmark_java_proto_lite",
"//mediapipe/java/com/google/mediapipe/framework:android_framework",
"@com_google_protobuf//:protobuf_javalite",
],
)
mediapipe/examples/coral/demo_run_graph_main.cc
+8 -7
View File
@@ -40,10 +40,11 @@ DEFINE_string(output_video_path, "",
"Full path of where to save result (.mp4 only). "
"If not provided, show result in a window.");
mediapipe::Status RunMPPGraph() {
absl::Status RunMPPGraph() {
std::string calculator_graph_config_contents;
MP_RETURN_IF_ERROR(mediapipe::file::GetContents(
FLAGS_calculator_graph_config_file, &calculator_graph_config_contents));
absl::GetFlag(FLAGS_calculator_graph_config_file),
&calculator_graph_config_contents));
LOG(INFO) << "Get calculator graph config contents: "
<< calculator_graph_config_contents;
mediapipe::CalculatorGraphConfig config =
@@ -56,22 +57,22 @@ mediapipe::Status RunMPPGraph() {
LOG(INFO) << "Initialize the camera or load the video.";
cv::VideoCapture capture;
const bool load_video = !FLAGS_input_video_path.empty();
const bool load_video = !absl::GetFlag(FLAGS_input_video_path).empty();
if (load_video) {
capture.open(FLAGS_input_video_path);
capture.open(absl::GetFlag(FLAGS_input_video_path));
} else {
capture.open(0);
}
RET_CHECK(capture.isOpened());
cv::VideoWriter writer;
const bool save_video = !FLAGS_output_video_path.empty();
const bool save_video = !absl::GetFlag(FLAGS_output_video_path).empty();
if (save_video) {
LOG(INFO) << "Prepare video writer.";
cv::Mat test_frame;
capture.read(test_frame); // Consume first frame.
capture.set(cv::CAP_PROP_POS_AVI_RATIO, 0); // Rewind to beginning.
writer.open(FLAGS_output_video_path,
writer.open(absl::GetFlag(FLAGS_output_video_path),
mediapipe::fourcc('a', 'v', 'c', '1'), // .mp4
capture.get(cv::CAP_PROP_FPS), test_frame.size());
RET_CHECK(writer.isOpened());
@@ -143,7 +144,7 @@ mediapipe::Status RunMPPGraph() {
int main(int argc, char** argv) {
google::InitGoogleLogging(argv[0]);
gflags::ParseCommandLineFlags(&argc, &argv, true);
mediapipe::Status run_status = RunMPPGraph();
absl::Status run_status = RunMPPGraph();
if (!run_status.ok()) {
LOG(ERROR) << "Failed to run the graph: " << run_status.message();
return EXIT_FAILURE;
mediapipe/examples/desktop/autoflip/calculators/BUILD
+1
View File
@@ -368,6 +368,7 @@ cc_test(
"//mediapipe/framework/deps:file_path",
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/formats:image_frame_opencv",
"//mediapipe/framework/port:commandlineflags",
"//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:opencv_core",
"//mediapipe/framework/port:opencv_imgcodecs",
mediapipe/examples/desktop/autoflip/calculators/border_detection_calculator.cc
+12 -13
View File
@@ -68,9 +68,9 @@ class BorderDetectionCalculator : public CalculatorBase {
BorderDetectionCalculator& operator=(const BorderDetectionCalculator&) =
delete;
static mediapipe::Status GetContract(mediapipe::CalculatorContract* cc);
mediapipe::Status Open(mediapipe::CalculatorContext* cc) override;
mediapipe::Status Process(mediapipe::CalculatorContext* cc) override;
static absl::Status GetContract(mediapipe::CalculatorContract* cc);
absl::Status Open(mediapipe::CalculatorContext* cc) override;
absl::Status Process(mediapipe::CalculatorContext* cc) override;
private:
// Given a color and image direction, check to see if a border of that color
@@ -83,7 +83,7 @@ class BorderDetectionCalculator : public CalculatorBase {
double ColorCount(const Color& mask_color, const cv::Mat& image) const;
// Set member vars (image size) and confirm no changes frame-to-frame.
mediapipe::Status SetAndCheckInputs(const cv::Mat& frame);
absl::Status SetAndCheckInputs(const cv::Mat& frame);
// Find the dominant color for a input image.
double FindDominantColor(const cv::Mat& image, Color* dominant_color);
@@ -97,15 +97,14 @@ class BorderDetectionCalculator : public CalculatorBase {
};
REGISTER_CALCULATOR(BorderDetectionCalculator);
mediapipe::Status BorderDetectionCalculator::Open(
mediapipe::CalculatorContext* cc) {
absl::Status BorderDetectionCalculator::Open(mediapipe::CalculatorContext* cc) {
options_ = cc->Options<BorderDetectionCalculatorOptions>();
RET_CHECK_LT(options_.vertical_search_distance(), 0.5)
<< "Search distance must be less than half the full image.";
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status BorderDetectionCalculator::SetAndCheckInputs(
absl::Status BorderDetectionCalculator::SetAndCheckInputs(
const cv::Mat& frame) {
if (frame_width_ < 0) {
frame_width_ = frame.cols;
@@ -118,10 +117,10 @@ mediapipe::Status BorderDetectionCalculator::SetAndCheckInputs(
RET_CHECK_EQ(frame.rows, frame_height_)
<< "Input frame dimensions must remain constant throughout the video.";
RET_CHECK_EQ(frame.channels(), 3) << "Input video type must be 3-channel";
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status BorderDetectionCalculator::Process(
absl::Status BorderDetectionCalculator::Process(
mediapipe::CalculatorContext* cc) {
if (!cc->Inputs().HasTag(kVideoInputTag) ||
cc->Inputs().Tag(kVideoInputTag).Value().IsEmpty()) {
@@ -173,7 +172,7 @@ mediapipe::Status BorderDetectionCalculator::Process(
.Tag(kDetectedBorders)
.AddPacket(Adopt(features.release()).At(cc->InputTimestamp()));
return mediapipe::OkStatus();
return absl::OkStatus();
}
// Find the dominant color within an image.
@@ -291,11 +290,11 @@ void BorderDetectionCalculator::DetectBorder(
}
}
mediapipe::Status BorderDetectionCalculator::GetContract(
absl::Status BorderDetectionCalculator::GetContract(
mediapipe::CalculatorContract* cc) {
cc->Inputs().Tag(kVideoInputTag).Set<ImageFrame>();
cc->Outputs().Tag(kDetectedBorders).Set<StaticFeatures>();
return mediapipe::OkStatus();
return absl::OkStatus();
}
} // namespace autoflip
mediapipe/examples/desktop/autoflip/calculators/content_zooming_calculator.cc
+130 -71
View File
@@ -55,23 +55,25 @@ class ContentZoomingCalculator : public CalculatorBase {
ContentZoomingCalculator(const ContentZoomingCalculator&) = delete;
ContentZoomingCalculator& operator=(const ContentZoomingCalculator&) = delete;
static mediapipe::Status GetContract(mediapipe::CalculatorContract* cc);
mediapipe::Status Open(mediapipe::CalculatorContext* cc) override;
mediapipe::Status Process(mediapipe::CalculatorContext* cc) override;
static absl::Status GetContract(mediapipe::CalculatorContract* cc);
absl::Status Open(mediapipe::CalculatorContext* cc) override;
absl::Status Process(mediapipe::CalculatorContext* cc) override;
private:
// Converts bounds to tilt offset, pan offset and height.
mediapipe::Status ConvertToPanTiltZoom(float xmin, float xmax, float ymin,
float ymax, int* tilt_offset,
int* pan_offset, int* height);
absl::Status ConvertToPanTiltZoom(float xmin, float xmax, float ymin,
float ymax, int* tilt_offset,
int* pan_offset, int* height);
// Sets max_frame_value_ and target_aspect_
absl::Status UpdateAspectAndMax();
ContentZoomingCalculatorOptions options_;
// Detection frame width/height.
int frame_height_;
int frame_width_;
// Path solver used to smooth top/bottom border crop values.
std::unique_ptr<KinematicPathSolver> path_solver_height_;
std::unique_ptr<KinematicPathSolver> path_solver_width_;
std::unique_ptr<KinematicPathSolver> path_solver_offset_;
std::unique_ptr<KinematicPathSolver> path_solver_zoom_;
std::unique_ptr<KinematicPathSolver> path_solver_pan_;
std::unique_ptr<KinematicPathSolver> path_solver_tilt_;
// Are parameters initialized.
bool initialized_;
// Stores the time of the last "only_required" input.
@@ -89,7 +91,7 @@ class ContentZoomingCalculator : public CalculatorBase {
};
REGISTER_CALCULATOR(ContentZoomingCalculator);
mediapipe::Status ContentZoomingCalculator::GetContract(
absl::Status ContentZoomingCalculator::GetContract(
mediapipe::CalculatorContract* cc) {
RET_CHECK(
!(cc->Inputs().HasTag(kVideoFrame) && cc->Inputs().HasTag(kVideoSize)))
@@ -114,11 +116,10 @@ mediapipe::Status ContentZoomingCalculator::GetContract(
if (cc->Outputs().HasTag(kCropRect)) {
cc->Outputs().Tag(kCropRect).Set<mediapipe::Rect>();
}
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status ContentZoomingCalculator::Open(
mediapipe::CalculatorContext* cc) {
absl::Status ContentZoomingCalculator::Open(mediapipe::CalculatorContext* cc) {
options_ = cc->Options<ContentZoomingCalculatorOptions>();
if (options_.has_kinematic_options()) {
return mediapipe::UnknownErrorBuilder(MEDIAPIPE_LOC)
@@ -131,10 +132,10 @@ mediapipe::Status ContentZoomingCalculator::Open(
"in kinematic_options_zoom and kinematic_options_tilt "
"directly.";
}
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status ContentZoomingCalculator::ConvertToPanTiltZoom(
absl::Status ContentZoomingCalculator::ConvertToPanTiltZoom(
float xmin, float xmax, float ymin, float ymax, int* tilt_offset,
int* pan_offset, int* height) {
// Find center of the y-axis offset (for tilt control).
@@ -142,10 +143,11 @@ mediapipe::Status ContentZoomingCalculator::ConvertToPanTiltZoom(
// Find center of the x-axis offset (for pan control).
float x_center = xmin + (xmax - xmin) / 2;
// Find size and apply scale factor to y-axis.
float fit_size = fmax((ymax - ymin) / options_.scale_factor(), xmax - xmin);
float fit_size_raw =
fmax((ymax - ymin) / options_.scale_factor(), xmax - xmin);
// Apply max frame for cases where the target size is different than input
// frame size.
fit_size = fmin(max_frame_value_, fit_size);
float fit_size = fmin(max_frame_value_, fit_size_raw);
// Prevent box from extending beyond the image.
if (y_center - fit_size / 2 < 0) {
y_center = fit_size / 2;
@@ -160,8 +162,8 @@ mediapipe::Status ContentZoomingCalculator::ConvertToPanTiltZoom(
// Scale to pixel coordinates.
*tilt_offset = frame_height_ * y_center;
*pan_offset = frame_width_ * x_center;
*height = frame_height_ * fit_size;
return mediapipe::OkStatus();
*height = frame_height_ * fit_size_raw;
return absl::OkStatus();
}
namespace {
@@ -185,10 +187,10 @@ mediapipe::autoflip::RectF ShiftDetection(
relative_bounding_box.width() * x_offset_percent);
return shifted_bb;
}
mediapipe::Status UpdateRanges(const SalientRegion& region,
const float shift_vertical,
const float shift_horizontal, float* xmin,
float* xmax, float* ymin, float* ymax) {
absl::Status UpdateRanges(const SalientRegion& region,
const float shift_vertical,
const float shift_horizontal, float* xmin,
float* xmax, float* ymin, float* ymax) {
if (!region.has_location_normalized()) {
return mediapipe::UnknownErrorBuilder(MEDIAPIPE_LOC)
<< "SalientRegion did not have location normalized set.";
@@ -200,12 +202,12 @@ mediapipe::Status UpdateRanges(const SalientRegion& region,
*ymin = fmin(*ymin, location.y());
*ymax = fmax(*ymax, location.y() + location.height());
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status UpdateRanges(const mediapipe::Detection& detection,
const float shift_vertical,
const float shift_horizontal, float* xmin,
float* xmax, float* ymin, float* ymax) {
absl::Status UpdateRanges(const mediapipe::Detection& detection,
const float shift_vertical,
const float shift_horizontal, float* xmin,
float* xmax, float* ymin, float* ymax) {
RET_CHECK(detection.location_data().format() ==
mediapipe::LocationData::RELATIVE_BOUNDING_BOX)
<< "Face detection input is lacking required relative_bounding_box()";
@@ -217,7 +219,7 @@ mediapipe::Status UpdateRanges(const mediapipe::Detection& detection,
*ymin = fmin(*ymin, location.ymin());
*ymax = fmax(*ymax, location.ymin() + location.height());
return mediapipe::OkStatus();
return absl::OkStatus();
}
void MakeStaticFeatures(const int top_border, const int bottom_border,
const int frame_width, const int frame_height,
@@ -236,57 +238,97 @@ void MakeStaticFeatures(const int top_border, const int bottom_border,
border_bottom->mutable_border_position()->set_width(frame_width);
border_bottom->mutable_border_position()->set_height(bottom_border);
}
} // namespace
mediapipe::Status ContentZoomingCalculator::Process(
mediapipe::CalculatorContext* cc) {
absl::Status GetVideoResolution(mediapipe::CalculatorContext* cc,
int* frame_width, int* frame_height) {
if (cc->Inputs().HasTag(kVideoFrame)) {
frame_width_ = cc->Inputs().Tag(kVideoFrame).Get<ImageFrame>().Width();
frame_height_ = cc->Inputs().Tag(kVideoFrame).Get<ImageFrame>().Height();
*frame_width = cc->Inputs().Tag(kVideoFrame).Get<ImageFrame>().Width();
*frame_height = cc->Inputs().Tag(kVideoFrame).Get<ImageFrame>().Height();
} else if (cc->Inputs().HasTag(kVideoSize)) {
if (cc->Inputs().Tag(kVideoSize).IsEmpty()) {
return mediapipe::OkStatus();
}
frame_width_ =
*frame_width =
cc->Inputs().Tag(kVideoSize).Get<std::pair<int, int>>().first;
frame_height_ =
*frame_height =
cc->Inputs().Tag(kVideoSize).Get<std::pair<int, int>>().second;
} else {
return mediapipe::UnknownErrorBuilder(MEDIAPIPE_LOC)
<< "Input VIDEO or VIDEO_SIZE must be provided.";
}
return absl::OkStatus();
}
} // namespace
absl::Status ContentZoomingCalculator::UpdateAspectAndMax() {
max_frame_value_ = 1.0;
target_aspect_ = frame_width_ / static_cast<float>(frame_height_);
// If target size is set and wider than input aspect, make sure to always
// crop the min required amount.
if (options_.has_target_size()) {
RET_CHECK_GT(options_.target_size().width(), 0)
<< "Provided target width not valid.";
RET_CHECK_GT(options_.target_size().height(), 0)
<< "Provided target height not valid.";
float input_aspect = frame_width_ / static_cast<float>(frame_height_);
target_aspect_ = options_.target_size().width() /
static_cast<float>(options_.target_size().height());
max_frame_value_ =
std::min(input_aspect / target_aspect_, target_aspect_ / input_aspect);
}
return absl::OkStatus();
}
absl::Status ContentZoomingCalculator::Process(
mediapipe::CalculatorContext* cc) {
// For async subgraph support, return on empty video size packets.
if (cc->Inputs().HasTag(kVideoSize) &&
cc->Inputs().Tag(kVideoSize).IsEmpty()) {
return absl::OkStatus();
}
int frame_width, frame_height;
MP_RETURN_IF_ERROR(GetVideoResolution(cc, &frame_width, &frame_height));
// Init on first call.
if (!initialized_) {
path_solver_height_ = std::make_unique<KinematicPathSolver>(
options_.kinematic_options_zoom(), 0, frame_height_,
static_cast<float>(frame_height_) / kFieldOfView);
path_solver_width_ = std::make_unique<KinematicPathSolver>(
frame_width_ = frame_width;
frame_height_ = frame_height;
path_solver_pan_ = std::make_unique<KinematicPathSolver>(
options_.kinematic_options_pan(), 0, frame_width_,
static_cast<float>(frame_width_) / kFieldOfView);
path_solver_offset_ = std::make_unique<KinematicPathSolver>(
path_solver_tilt_ = std::make_unique<KinematicPathSolver>(
options_.kinematic_options_tilt(), 0, frame_height_,
static_cast<float>(frame_height_) / kFieldOfView);
max_frame_value_ = 1.0;
target_aspect_ = frame_width_ / static_cast<float>(frame_height_);
// If target size is set and wider than input aspect, make sure to always
// crop the min required amount.
if (options_.has_target_size()) {
RET_CHECK_GT(options_.target_size().width(), 0)
<< "Provided target width not valid.";
RET_CHECK_GT(options_.target_size().height(), 0)
<< "Provided target height not valid.";
float input_aspect = frame_width_ / static_cast<float>(frame_height_);
target_aspect_ = options_.target_size().width() /
static_cast<float>(options_.target_size().height());
max_frame_value_ = std::min(input_aspect / target_aspect_,
target_aspect_ / input_aspect);
}
MP_RETURN_IF_ERROR(UpdateAspectAndMax());
int min_zoom_size = frame_height_ * (options_.max_zoom_value_deg() /
static_cast<double>(kFieldOfView));
path_solver_zoom_ = std::make_unique<KinematicPathSolver>(
options_.kinematic_options_zoom(), min_zoom_size,
max_frame_value_ * frame_height_,
static_cast<float>(frame_height_) / kFieldOfView);
last_measured_height_ = max_frame_value_ * frame_height_;
last_measured_x_offset_ = target_aspect_ * frame_width_;
last_measured_y_offset_ = frame_width_ / 2;
initialized_ = true;
}
// Update state for change in input resolution.
if (frame_width_ != frame_width || frame_height_ != frame_height) {
double width_scale = frame_width / static_cast<double>(frame_width_);
double height_scale = frame_height / static_cast<double>(frame_height_);
last_measured_height_ = last_measured_height_ * height_scale;
last_measured_y_offset_ = last_measured_y_offset_ * height_scale;
last_measured_x_offset_ = last_measured_x_offset_ * width_scale;
frame_width_ = frame_width;
frame_height_ = frame_height;
MP_RETURN_IF_ERROR(UpdateAspectAndMax());
MP_RETURN_IF_ERROR(path_solver_pan_->UpdateMinMaxLocation(0, frame_width_));
MP_RETURN_IF_ERROR(
path_solver_tilt_->UpdateMinMaxLocation(0, frame_height_));
int min_zoom_size = frame_height_ * (options_.max_zoom_value_deg() /
static_cast<double>(kFieldOfView));
MP_RETURN_IF_ERROR(path_solver_zoom_->UpdateMinMaxLocation(
min_zoom_size, max_frame_value_ * frame_height_));
MP_RETURN_IF_ERROR(path_solver_zoom_->UpdatePixelsPerDegree(
static_cast<float>(frame_height_) / kFieldOfView));
}
bool only_required_found = false;
// Compute the box that contains all "is_required" detections.
@@ -307,11 +349,13 @@ mediapipe::Status ContentZoomingCalculator::Process(
if (cc->Inputs().HasTag(kDetections)) {
if (cc->Inputs().Tag(kDetections).IsEmpty()) {
auto default_rect = absl::make_unique<mediapipe::Rect>();
default_rect->set_x_center(frame_width_ / 2);
default_rect->set_y_center(frame_height_ / 2);
default_rect->set_width(frame_width_);
default_rect->set_height(frame_height_);
cc->Outputs().Tag(kCropRect).Add(default_rect.release(),
Timestamp(cc->InputTimestamp()));
return mediapipe::OkStatus();
return absl::OkStatus();
}
auto raw_detections =
cc->Inputs().Tag(kDetections).Get<std::vector<mediapipe::Detection>>();
@@ -350,31 +394,46 @@ mediapipe::Status ContentZoomingCalculator::Process(
offset_y = last_measured_y_offset_;
}
// Check if the camera is changing in pan, tilt or zoom. If the camera is in
// motion disable temporal filtering.
bool pan_state, tilt_state, zoom_state;
MP_RETURN_IF_ERROR(path_solver_pan_->PredictMotionState(
offset_x, cc->InputTimestamp().Microseconds(), &pan_state));
MP_RETURN_IF_ERROR(path_solver_tilt_->PredictMotionState(
offset_y, cc->InputTimestamp().Microseconds(), &tilt_state));
MP_RETURN_IF_ERROR(path_solver_zoom_->PredictMotionState(
height, cc->InputTimestamp().Microseconds(), &zoom_state));
if (pan_state || tilt_state || zoom_state) {
path_solver_pan_->ClearHistory();
path_solver_tilt_->ClearHistory();
path_solver_zoom_->ClearHistory();
}
// Compute smoothed zoom camera path.
MP_RETURN_IF_ERROR(path_solver_height_->AddObservation(
MP_RETURN_IF_ERROR(path_solver_zoom_->AddObservation(
height, cc->InputTimestamp().Microseconds()));
int path_height;
MP_RETURN_IF_ERROR(path_solver_height_->GetState(&path_height));
MP_RETURN_IF_ERROR(path_solver_zoom_->GetState(&path_height));
int path_width = path_height * target_aspect_;
// Update pixel-per-degree value for pan/tilt.
int target_height;
MP_RETURN_IF_ERROR(path_solver_height_->GetTargetPosition(&target_height));
MP_RETURN_IF_ERROR(path_solver_zoom_->GetTargetPosition(&target_height));
int target_width = target_height * target_aspect_;
MP_RETURN_IF_ERROR(path_solver_width_->UpdatePixelsPerDegree(
MP_RETURN_IF_ERROR(path_solver_pan_->UpdatePixelsPerDegree(
static_cast<float>(target_width) / kFieldOfView));
MP_RETURN_IF_ERROR(path_solver_offset_->UpdatePixelsPerDegree(
MP_RETURN_IF_ERROR(path_solver_tilt_->UpdatePixelsPerDegree(
static_cast<float>(target_height) / kFieldOfView));
// Compute smoothed pan/tilt paths.
MP_RETURN_IF_ERROR(path_solver_width_->AddObservation(
MP_RETURN_IF_ERROR(path_solver_pan_->AddObservation(
offset_x, cc->InputTimestamp().Microseconds()));
MP_RETURN_IF_ERROR(path_solver_offset_->AddObservation(
MP_RETURN_IF_ERROR(path_solver_tilt_->AddObservation(
offset_y, cc->InputTimestamp().Microseconds()));
int path_offset_x;
MP_RETURN_IF_ERROR(path_solver_width_->GetState(&path_offset_x));
MP_RETURN_IF_ERROR(path_solver_pan_->GetState(&path_offset_x));
int path_offset_y;
MP_RETURN_IF_ERROR(path_solver_offset_->GetState(&path_offset_y));
MP_RETURN_IF_ERROR(path_solver_tilt_->GetState(&path_offset_y));
// Prevent box from extending beyond the image after camera smoothing.
if (path_offset_y - ceil(path_height / 2.0) < 0) {
@@ -415,7 +474,7 @@ mediapipe::Status ContentZoomingCalculator::Process(
Timestamp(cc->InputTimestamp()));
}
return mediapipe::OkStatus();
return absl::OkStatus();
}
} // namespace autoflip
mediapipe/examples/desktop/autoflip/calculators/content_zooming_calculator.proto
+4 -1
View File
@@ -19,7 +19,7 @@ package mediapipe.autoflip;
import "mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver.proto";
import "mediapipe/framework/calculator.proto";
// NextTag: 13
// NextTag: 14
message ContentZoomingCalculatorOptions {
extend mediapipe.CalculatorOptions {
optional ContentZoomingCalculatorOptions ext = 313091992;
@@ -52,6 +52,9 @@ message ContentZoomingCalculatorOptions {
optional float detection_shift_vertical = 11 [default = 0.0];
optional float detection_shift_horizontal = 12 [default = 0.0];
// Defines the smallest value in degrees the camera is permitted to zoom.
optional float max_zoom_value_deg = 13 [default = 35];
// Deprecated parameters
optional KinematicOptions kinematic_options = 2 [deprecated = true];
optional int64 min_motion_to_reframe = 4 [deprecated = true];
mediapipe/examples/desktop/autoflip/calculators/content_zooming_calculator_test.cc
+108 -4
View File
@@ -42,6 +42,20 @@ const char kConfigA[] = R"(
input_stream: "VIDEO:camera_frames"
input_stream: "SALIENT_REGIONS:detection_set"
output_stream: "BORDERS:borders"
options: {
[mediapipe.autoflip.ContentZoomingCalculatorOptions.ext]: {
max_zoom_value_deg: 0
kinematic_options_zoom {
min_motion_to_reframe: 1.2
}
kinematic_options_tilt {
min_motion_to_reframe: 1.2
}
kinematic_options_pan {
min_motion_to_reframe: 1.2
}
}
}
)";
const char kConfigB[] = R"(
@@ -55,6 +69,16 @@ const char kConfigB[] = R"(
width: 1000
height: 500
}
max_zoom_value_deg: 0
kinematic_options_zoom {
min_motion_to_reframe: 1.2
}
kinematic_options_tilt {
min_motion_to_reframe: 1.2
}
kinematic_options_pan {
min_motion_to_reframe: 1.2
}
}
}
)";
@@ -64,6 +88,20 @@ const char kConfigC[] = R"(
input_stream: "VIDEO_SIZE:size"
input_stream: "SALIENT_REGIONS:detection_set"
output_stream: "BORDERS:borders"
options: {
[mediapipe.autoflip.ContentZoomingCalculatorOptions.ext]: {
max_zoom_value_deg: 0
kinematic_options_zoom {
min_motion_to_reframe: 1.2
}
kinematic_options_tilt {
min_motion_to_reframe: 1.2
}
kinematic_options_pan {
min_motion_to_reframe: 1.2
}
}
}
)";
const char kConfigD[] = R"(
@@ -71,6 +109,20 @@ const char kConfigD[] = R"(
input_stream: "VIDEO_SIZE:size"
input_stream: "DETECTIONS:detections"
output_stream: "CROP_RECT:rect"
options: {
[mediapipe.autoflip.ContentZoomingCalculatorOptions.ext]: {
max_zoom_value_deg: 0
kinematic_options_zoom {
min_motion_to_reframe: 1.2
}
kinematic_options_tilt {
min_motion_to_reframe: 1.2
}
kinematic_options_pan {
min_motion_to_reframe: 1.2
}
}
}
)";
void CheckBorder(const StaticFeatures& static_features, int width, int height,
@@ -91,8 +143,9 @@ void CheckBorder(const StaticFeatures& static_features, int width, int height,
EXPECT_EQ(Border::BOTTOM, part.relative_position());
}
void AddDetection(const cv::Rect_<float>& position, const int64 time,
CalculatorRunner* runner) {
void AddDetectionFrameSize(const cv::Rect_<float>& position, const int64 time,
const int width, const int height,
CalculatorRunner* runner) {
auto detections = std::make_unique<std::vector<mediapipe::Detection>>();
mediapipe::Detection detection;
detection.mutable_location_data()->set_format(
@@ -111,12 +164,17 @@ void AddDetection(const cv::Rect_<float>& position, const int64 time,
->Tag("DETECTIONS")
.packets.push_back(Adopt(detections.release()).At(Timestamp(time)));
auto input_size = ::absl::make_unique<std::pair<int, int>>(1000, 1000);
auto input_size = ::absl::make_unique<std::pair<int, int>>(width, height);
runner->MutableInputs()
->Tag("VIDEO_SIZE")
.packets.push_back(Adopt(input_size.release()).At(Timestamp(time)));
}
void AddDetection(const cv::Rect_<float>& position, const int64 time,
CalculatorRunner* runner) {
AddDetectionFrameSize(position, time, 1000, 1000, runner);
}
void CheckCropRect(const int x_center, const int y_center, const int width,
const int height, const int frame_number,
const std::vector<Packet>& output_packets) {
@@ -433,7 +491,53 @@ TEST(ContentZoomingCalculatorTest, EmptyDetections) {
->Tag("VIDEO_SIZE")
.packets.push_back(Adopt(input_size.release()).At(Timestamp(0)));
MP_ASSERT_OK(runner->Run());
CheckCropRect(0, 0, 1000, 1000, 0,
CheckCropRect(500, 500, 1000, 1000, 0,
runner->Outputs().Tag("CROP_RECT").packets);
}
TEST(ContentZoomingCalculatorTest, ResolutionChangeStationary) {
auto config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfigD);
auto runner = ::absl::make_unique<CalculatorRunner>(config);
AddDetectionFrameSize(cv::Rect_<float>(.4, .4, .2, .2), 0, 1000, 1000,
runner.get());
AddDetectionFrameSize(cv::Rect_<float>(.4, .4, .2, .2), 1, 500, 500,
runner.get());
MP_ASSERT_OK(runner->Run());
CheckCropRect(500, 500, 222, 222, 0,
runner->Outputs().Tag("CROP_RECT").packets);
CheckCropRect(500 * 0.5, 500 * 0.5, 222 * 0.5, 222 * 0.5, 1,
runner->Outputs().Tag("CROP_RECT").packets);
}
TEST(ContentZoomingCalculatorTest, ResolutionChangeZooming) {
auto config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfigD);
auto runner = ::absl::make_unique<CalculatorRunner>(config);
AddDetectionFrameSize(cv::Rect_<float>(.1, .1, .8, .8), 0, 1000, 1000,
runner.get());
AddDetectionFrameSize(cv::Rect_<float>(.4, .4, .2, .2), 1000000, 1000, 1000,
runner.get());
AddDetectionFrameSize(cv::Rect_<float>(.4, .4, .2, .2), 2000000, 500, 500,
runner.get());
MP_ASSERT_OK(runner->Run());
CheckCropRect(500, 500, 888, 888, 0,
runner->Outputs().Tag("CROP_RECT").packets);
CheckCropRect(500, 500, 588, 588, 1,
runner->Outputs().Tag("CROP_RECT").packets);
CheckCropRect(500 * 0.5, 500 * 0.5, 288 * 0.5, 288 * 0.5, 2,
runner->Outputs().Tag("CROP_RECT").packets);
}
TEST(ContentZoomingCalculatorTest, MaxZoomValue) {
auto config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfigD);
auto* options = config.mutable_options()->MutableExtension(
ContentZoomingCalculatorOptions::ext);
options->set_max_zoom_value_deg(55);
auto runner = ::absl::make_unique<CalculatorRunner>(config);
AddDetectionFrameSize(cv::Rect_<float>(.4, .4, .2, .2), 0, 1000, 1000,
runner.get());
MP_ASSERT_OK(runner->Run());
// 55/60 * 1000 = 916
CheckCropRect(500, 500, 916, 916, 0,
runner->Outputs().Tag("CROP_RECT").packets);
}
mediapipe/examples/desktop/autoflip/calculators/face_box_adjuster_calculator.proto
+50
View File
@@ -0,0 +1,50 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto2";
package mediapipe.autoflip;
import "mediapipe/framework/calculator.proto";
message FaceBoxAdjusterCalculatorOptions {
extend mediapipe.CalculatorOptions {
optional FaceBoxAdjusterCalculatorOptions ext = 347462240;
}
// When faces are detected in a given frame, we check these number of frames
// in the past. We include only those faces in auto framing that have been
// seen in this past history. This helps reduce False Positives and also
// handles some of the edge cases. Setting the value to 0 disables the
// feature.
optional int32 num_frame_history = 1 [default = 0];
// IOU threshold for matching detected faces with the faces in the frame
// history buffer.
optional float iou_threshold = 2 [default = 0.2];
// If true, the face boxes are adjusted based on their face pose. This is done
// to correct for extreme poses that can cause the detected face boxes to be
// either too big or too small.
optional bool adjust_for_pose = 3 [default = true];
// There are DEPRECATED fields. Do not use.
optional float box_area_change_per_up_tilt_degree = 4 [deprecated = true];
optional float box_area_change_per_down_tilt_degree = 5 [deprecated = true];
// The ratios of the face-pose corrected IPD to the face bounding box's width
// and height respectively.
optional float ipd_face_box_width_ratio = 6 [default = 0.5566];
optional float ipd_face_box_height_ratio = 7 [default = 0.3131];
}
mediapipe/examples/desktop/autoflip/calculators/face_to_region_calculator.cc
+9 -11
View File
@@ -55,9 +55,9 @@ class FaceToRegionCalculator : public CalculatorBase {
FaceToRegionCalculator(const FaceToRegionCalculator&) = delete;
FaceToRegionCalculator& operator=(const FaceToRegionCalculator&) = delete;
static mediapipe::Status GetContract(mediapipe::CalculatorContract* cc);
mediapipe::Status Open(mediapipe::CalculatorContext* cc) override;
mediapipe::Status Process(mediapipe::CalculatorContext* cc) override;
static absl::Status GetContract(mediapipe::CalculatorContract* cc);
absl::Status Open(mediapipe::CalculatorContext* cc) override;
absl::Status Process(mediapipe::CalculatorContext* cc) override;
private:
double NormalizeX(const int pixel);
@@ -78,18 +78,17 @@ REGISTER_CALCULATOR(FaceToRegionCalculator);
FaceToRegionCalculator::FaceToRegionCalculator() {}
mediapipe::Status FaceToRegionCalculator::GetContract(
absl::Status FaceToRegionCalculator::GetContract(
mediapipe::CalculatorContract* cc) {
if (cc->Inputs().HasTag("VIDEO")) {
cc->Inputs().Tag("VIDEO").Set<ImageFrame>();
}
cc->Inputs().Tag("FACES").Set<std::vector<mediapipe::Detection>>();
cc->Outputs().Tag("REGIONS").Set<DetectionSet>();
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status FaceToRegionCalculator::Open(
mediapipe::CalculatorContext* cc) {
absl::Status FaceToRegionCalculator::Open(mediapipe::CalculatorContext* cc) {
options_ = cc->Options<FaceToRegionCalculatorOptions>();
if (!cc->Inputs().HasTag("VIDEO")) {
RET_CHECK(!options_.use_visual_scorer())
@@ -105,7 +104,7 @@ mediapipe::Status FaceToRegionCalculator::Open(
scorer_ = absl::make_unique<VisualScorer>(options_.scorer_options());
frame_width_ = -1;
frame_height_ = -1;
return mediapipe::OkStatus();
return absl::OkStatus();
}
inline double FaceToRegionCalculator::NormalizeX(const int pixel) {
@@ -146,8 +145,7 @@ void FaceToRegionCalculator::ExtendSalientRegionWithPoint(
}
}
mediapipe::Status FaceToRegionCalculator::Process(
mediapipe::CalculatorContext* cc) {
absl::Status FaceToRegionCalculator::Process(mediapipe::CalculatorContext* cc) {
if (cc->Inputs().HasTag("VIDEO") &&
cc->Inputs().Tag("VIDEO").Value().IsEmpty()) {
return mediapipe::UnknownErrorBuilder(MEDIAPIPE_LOC)
@@ -280,7 +278,7 @@ mediapipe::Status FaceToRegionCalculator::Process(
}
cc->Outputs().Tag("REGIONS").Add(region_set.release(), cc->InputTimestamp());
return mediapipe::OkStatus();
return absl::OkStatus();
}
} // namespace autoflip
mediapipe/examples/desktop/autoflip/calculators/localization_to_region_calculator.cc
+9 -9
View File
@@ -38,9 +38,9 @@ class LocalizationToRegionCalculator : public mediapipe::CalculatorBase {
LocalizationToRegionCalculator& operator=(
const LocalizationToRegionCalculator&) = delete;
static mediapipe::Status GetContract(mediapipe::CalculatorContract* cc);
mediapipe::Status Open(mediapipe::CalculatorContext* cc) override;
mediapipe::Status Process(mediapipe::CalculatorContext* cc) override;
static absl::Status GetContract(mediapipe::CalculatorContract* cc);
absl::Status Open(mediapipe::CalculatorContext* cc) override;
absl::Status Process(mediapipe::CalculatorContext* cc) override;
private:
// Calculator options.
@@ -84,21 +84,21 @@ void FillSalientRegion(const mediapipe::Detection& detection,
} // namespace
mediapipe::Status LocalizationToRegionCalculator::GetContract(
absl::Status LocalizationToRegionCalculator::GetContract(
mediapipe::CalculatorContract* cc) {
cc->Inputs().Tag("DETECTIONS").Set<std::vector<mediapipe::Detection>>();
cc->Outputs().Tag("REGIONS").Set<DetectionSet>();
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status LocalizationToRegionCalculator::Open(
absl::Status LocalizationToRegionCalculator::Open(
mediapipe::CalculatorContext* cc) {
options_ = cc->Options<LocalizationToRegionCalculatorOptions>();
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status LocalizationToRegionCalculator::Process(
absl::Status LocalizationToRegionCalculator::Process(
mediapipe::CalculatorContext* cc) {
const auto& annotations =
cc->Inputs().Tag("DETECTIONS").Get<std::vector<mediapipe::Detection>>();
@@ -119,7 +119,7 @@ mediapipe::Status LocalizationToRegionCalculator::Process(
}
cc->Outputs().Tag("REGIONS").Add(regions.release(), cc->InputTimestamp());
return mediapipe::OkStatus();
return absl::OkStatus();
}
} // namespace autoflip
mediapipe/examples/desktop/autoflip/calculators/scene_cropping_calculator.cc
+27 -29
View File
@@ -68,7 +68,7 @@ constexpr char kOutputSummary[] = "CROPPING_SUMMARY";
constexpr char kExternalRenderingPerFrame[] = "EXTERNAL_RENDERING_PER_FRAME";
constexpr char kExternalRenderingFullVid[] = "EXTERNAL_RENDERING_FULL_VID";
mediapipe::Status SceneCroppingCalculator::GetContract(
absl::Status SceneCroppingCalculator::GetContract(
mediapipe::CalculatorContract* cc) {
if (cc->InputSidePackets().HasTag(kInputExternalSettings)) {
cc->InputSidePackets().Tag(kInputExternalSettings).Set<std::string>();
@@ -136,10 +136,10 @@ mediapipe::Status SceneCroppingCalculator::GetContract(
cc->Outputs().HasTag(kExternalRenderingFullVid) ||
cc->Outputs().HasTag(kOutputCroppedFrames))
<< "At leaset one output stream must be specified";
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status SceneCroppingCalculator::Open(CalculatorContext* cc) {
absl::Status SceneCroppingCalculator::Open(CalculatorContext* cc) {
options_ = cc->Options<SceneCroppingCalculatorOptions>();
RET_CHECK_GT(options_.max_scene_size(), 0)
<< "Maximum scene size is non-positive.";
@@ -175,17 +175,17 @@ mediapipe::Status SceneCroppingCalculator::Open(CalculatorContext* cc) {
should_perform_frame_cropping_ = cc->Outputs().HasTag(kOutputCroppedFrames);
scene_camera_motion_analyzer_ = absl::make_unique<SceneCameraMotionAnalyzer>(
options_.scene_camera_motion_analyzer_options());
return mediapipe::OkStatus();
return absl::OkStatus();
}
namespace {
mediapipe::Status ParseAspectRatioString(const std::string& aspect_ratio_string,
double* aspect_ratio) {
absl::Status ParseAspectRatioString(const std::string& aspect_ratio_string,
double* aspect_ratio) {
std::string error_msg =
"Aspect ratio std::string must be in the format of 'width:height', e.g. "
"'1:1' or '5:4', your input was " +
aspect_ratio_string;
auto pos = aspect_ratio_string.find(":");
auto pos = aspect_ratio_string.find(':');
RET_CHECK(pos != std::string::npos) << error_msg;
double width_ratio;
RET_CHECK(absl::SimpleAtod(aspect_ratio_string.substr(0, pos), &width_ratio))
@@ -196,7 +196,7 @@ mediapipe::Status ParseAspectRatioString(const std::string& aspect_ratio_string,
&height_ratio))
<< error_msg;
*aspect_ratio = width_ratio / height_ratio;
return mediapipe::OkStatus();
return absl::OkStatus();
}
void ConstructExternalRenderMessage(
const cv::Rect& crop_from_location, const cv::Rect& render_to_location,
@@ -235,7 +235,7 @@ int RoundToEven(float value) {
} // namespace
mediapipe::Status SceneCroppingCalculator::InitializeSceneCroppingCalculator(
absl::Status SceneCroppingCalculator::InitializeSceneCroppingCalculator(
mediapipe::CalculatorContext* cc) {
if (cc->Inputs().HasTag(kInputVideoFrames)) {
const auto& frame = cc->Inputs().Tag(kInputVideoFrames).Get<ImageFrame>();
@@ -302,8 +302,7 @@ mediapipe::Status SceneCroppingCalculator::InitializeSceneCroppingCalculator(
target_height_ = frame_height_;
break;
case SceneCroppingCalculatorOptions::UNKNOWN:
return mediapipe::InvalidArgumentError(
"target_size_type not set properly.");
return absl::InvalidArgumentError("target_size_type not set properly.");
}
target_aspect_ratio_ = GetRatio(target_width_, target_height_);
@@ -337,7 +336,7 @@ mediapipe::Status SceneCroppingCalculator::InitializeSceneCroppingCalculator(
scene_cropper_ = absl::make_unique<SceneCropper>(
options_.camera_motion_options(), frame_width_, frame_height_);
return mediapipe::OkStatus();
return absl::OkStatus();
}
bool HasFrameSignal(mediapipe::CalculatorContext* cc) {
@@ -347,7 +346,7 @@ bool HasFrameSignal(mediapipe::CalculatorContext* cc) {
return !cc->Inputs().Tag(kInputVideoSize).Value().IsEmpty();
}
mediapipe::Status SceneCroppingCalculator::Process(
absl::Status SceneCroppingCalculator::Process(
mediapipe::CalculatorContext* cc) {
// Sets frame dimension and initializes scenecroppingcalculator on first video
// frame.
@@ -417,11 +416,10 @@ mediapipe::Status SceneCroppingCalculator::Process(
continue_last_scene_ = true;
}
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status SceneCroppingCalculator::Close(
mediapipe::CalculatorContext* cc) {
absl::Status SceneCroppingCalculator::Close(mediapipe::CalculatorContext* cc) {
if (!scene_frame_timestamps_.empty()) {
MP_RETURN_IF_ERROR(ProcessScene(/* is_end_of_scene = */ true, cc));
}
@@ -435,12 +433,12 @@ mediapipe::Status SceneCroppingCalculator::Close(
.Tag(kExternalRenderingFullVid)
.Add(external_render_list_.release(), Timestamp::PostStream());
}
return mediapipe::OkStatus();
return absl::OkStatus();
}
// TODO: split this function into two, one for calculating the border
// sizes, the other for the actual removal of borders from the frames.
mediapipe::Status SceneCroppingCalculator::RemoveStaticBorders(
absl::Status SceneCroppingCalculator::RemoveStaticBorders(
CalculatorContext* cc, int* top_border_size, int* bottom_border_size) {
*top_border_size = 0;
*bottom_border_size = 0;
@@ -492,10 +490,10 @@ mediapipe::Status SceneCroppingCalculator::RemoveStaticBorders(
*key_frame_infos_[i].mutable_detections() = adjusted_detections;
}
}
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status SceneCroppingCalculator::InitializeFrameCropRegionComputer() {
absl::Status SceneCroppingCalculator::InitializeFrameCropRegionComputer() {
key_frame_crop_options_ = options_.key_frame_crop_options();
MP_RETURN_IF_ERROR(
SetKeyFrameCropTarget(frame_width_, effective_frame_height_,
@@ -504,7 +502,7 @@ mediapipe::Status SceneCroppingCalculator::InitializeFrameCropRegionComputer() {
VLOG(1) << "Target height " << key_frame_crop_options_.target_height();
frame_crop_region_computer_ =
absl::make_unique<FrameCropRegionComputer>(key_frame_crop_options_);
return mediapipe::OkStatus();
return absl::OkStatus();
}
void SceneCroppingCalculator::FilterKeyFrameInfo() {
@@ -530,8 +528,8 @@ void SceneCroppingCalculator::FilterKeyFrameInfo() {
}
}
mediapipe::Status SceneCroppingCalculator::ProcessScene(
const bool is_end_of_scene, CalculatorContext* cc) {
absl::Status SceneCroppingCalculator::ProcessScene(const bool is_end_of_scene,
CalculatorContext* cc) {
// Removes detections under special circumstances.
FilterKeyFrameInfo();
@@ -653,10 +651,10 @@ mediapipe::Status SceneCroppingCalculator::ProcessScene(
is_key_frames_.clear();
static_features_.clear();
static_features_timestamps_.clear();
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status SceneCroppingCalculator::FormatAndOutputCroppedFrames(
absl::Status SceneCroppingCalculator::FormatAndOutputCroppedFrames(
const int crop_width, const int crop_height, const int num_frames,
std::vector<cv::Rect>* render_to_locations, bool* apply_padding,
std::vector<cv::Scalar>* padding_colors, float* vertical_fill_percent,
@@ -729,7 +727,7 @@ mediapipe::Status SceneCroppingCalculator::FormatAndOutputCroppedFrames(
padding_colors->push_back(padding_color_to_add);
}
if (!cropped_frames_ptr) {
return mediapipe::OkStatus();
return absl::OkStatus();
}
// Resizes cropped frames, pads frames, and output frames.
@@ -772,10 +770,10 @@ mediapipe::Status SceneCroppingCalculator::FormatAndOutputCroppedFrames(
.Add(scaled_frame.release(), timestamp);
}
}
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status SceneCroppingCalculator::OutputVizFrames(
absl::Status SceneCroppingCalculator::OutputVizFrames(
const std::vector<KeyFrameCropResult>& key_frame_crop_results,
const std::vector<FocusPointFrame>& focus_point_frames,
const std::vector<cv::Rect>& crop_from_locations,
@@ -815,7 +813,7 @@ mediapipe::Status SceneCroppingCalculator::OutputVizFrames(
.Add(viz_frames[i].release(), Timestamp(scene_frame_timestamps_[i]));
}
}
return mediapipe::OkStatus();
return absl::OkStatus();
}
REGISTER_CALCULATOR(SceneCroppingCalculator);
mediapipe/examples/desktop/autoflip/calculators/scene_cropping_calculator.h
+11 -13
View File
@@ -125,35 +125,34 @@ namespace autoflip {
// fields are optional with default settings.
class SceneCroppingCalculator : public CalculatorBase {
public:
static mediapipe::Status GetContract(CalculatorContract* cc);
static absl::Status GetContract(CalculatorContract* cc);
// Validates calculator options and initializes SceneCameraMotionAnalyzer and
// SceneCropper.
mediapipe::Status Open(CalculatorContext* cc) override;
absl::Status Open(CalculatorContext* cc) override;
// Buffers each scene frame and its timestamp. Packs and stores KeyFrameInfo
// for key frames (a.k.a. frames with detection features). When a shot
// boundary is encountered or when the buffer is full, calls ProcessScene()
// to process the scene at once, and clears buffers.
mediapipe::Status Process(CalculatorContext* cc) override;
absl::Status Process(CalculatorContext* cc) override;
// Calls ProcessScene() on remaining buffered frames. Optionally outputs a
// VideoCroppingSummary if the output stream CROPPING_SUMMARY is present.
mediapipe::Status Close(mediapipe::CalculatorContext* cc) override;
absl::Status Close(mediapipe::CalculatorContext* cc) override;
private:
// Removes any static borders from the scene frames before cropping. The
// arguments |top_border_size| and |bottom_border_size| report the size of the
// removed borders.
mediapipe::Status RemoveStaticBorders(CalculatorContext* cc,
int* top_border_size,
int* bottom_border_size);
absl::Status RemoveStaticBorders(CalculatorContext* cc, int* top_border_size,
int* bottom_border_size);
// Sets up autoflip after first frame is received and input size is known.
mediapipe::Status InitializeSceneCroppingCalculator(
absl::Status InitializeSceneCroppingCalculator(
mediapipe::CalculatorContext* cc);
// Initializes a FrameCropRegionComputer given input and target frame sizes.
mediapipe::Status InitializeFrameCropRegionComputer();
absl::Status InitializeFrameCropRegionComputer();
// Processes a scene using buffered scene frames and KeyFrameInfos:
// 1. Computes key frame crop regions using a FrameCropRegionComputer.
@@ -165,8 +164,7 @@ class SceneCroppingCalculator : public CalculatorBase {
// to force flush).
// 6. Optionally outputs visualization frames.
// 7. Optionally updates cropping summary.
mediapipe::Status ProcessScene(const bool is_end_of_scene,
CalculatorContext* cc);
absl::Status ProcessScene(const bool is_end_of_scene, CalculatorContext* cc);
// Formats and outputs the cropped frames passed in through
// |cropped_frames_ptr|. Scales them to be at least as big as the target
@@ -177,14 +175,14 @@ class SceneCroppingCalculator : public CalculatorBase {
// cropped frames. This is useful when the calculator is only used for
// computing the cropping metadata rather than doing the actual cropping
// operation.
mediapipe::Status FormatAndOutputCroppedFrames(
absl::Status FormatAndOutputCroppedFrames(
const int crop_width, const int crop_height, const int num_frames,
std::vector<cv::Rect>* render_to_locations, bool* apply_padding,
std::vector<cv::Scalar>* padding_colors, float* vertical_fill_percent,
const std::vector<cv::Mat>* cropped_frames_ptr, CalculatorContext* cc);
// Draws and outputs visualization frames if those streams are present.
mediapipe::Status OutputVizFrames(
absl::Status OutputVizFrames(
const std::vector<KeyFrameCropResult>& key_frame_crop_results,
const std::vector<FocusPointFrame>& focus_point_frames,
const std::vector<cv::Rect>& crop_from_locations,
mediapipe/examples/desktop/autoflip/calculators/scene_cropping_calculator_test.cc
+2
View File
@@ -803,6 +803,7 @@ TEST(SceneCroppingCalculatorTest, OutputsCropMessageKinematicPath) {
SceneCroppingCalculatorOptions::ext);
auto* kinematic_options =
options->mutable_camera_motion_options()->mutable_kinematic_options();
kinematic_options->set_min_motion_to_reframe(1.2);
kinematic_options->set_max_velocity(200);
auto runner = absl::make_unique<CalculatorRunner>(config);
@@ -875,6 +876,7 @@ TEST(SceneCroppingCalculatorTest, OutputsCropMessageKinematicPathNoVideo) {
SceneCroppingCalculatorOptions::ext);
auto* kinematic_options =
options->mutable_camera_motion_options()->mutable_kinematic_options();
kinematic_options->set_min_motion_to_reframe(1.2);
kinematic_options->set_max_velocity(2.0);
auto runner = absl::make_unique<CalculatorRunner>(config);
mediapipe/examples/desktop/autoflip/calculators/shot_boundary_calculator.cc
+11 -13
View File
@@ -60,9 +60,9 @@ class ShotBoundaryCalculator : public mediapipe::CalculatorBase {
ShotBoundaryCalculator(const ShotBoundaryCalculator&) = delete;
ShotBoundaryCalculator& operator=(const ShotBoundaryCalculator&) = delete;
static mediapipe::Status GetContract(mediapipe::CalculatorContract* cc);
mediapipe::Status Open(mediapipe::CalculatorContext* cc) override;
mediapipe::Status Process(mediapipe::CalculatorContext* cc) override;
static absl::Status GetContract(mediapipe::CalculatorContract* cc);
absl::Status Open(mediapipe::CalculatorContext* cc) override;
absl::Status Process(mediapipe::CalculatorContext* cc) override;
private:
// Computes the histogram of an image.
@@ -98,12 +98,11 @@ void ShotBoundaryCalculator::ComputeHistogram(const cv::Mat& image,
kHistogramBinNum, kHistogramRange, true, false);
}
mediapipe::Status ShotBoundaryCalculator::Open(
mediapipe::CalculatorContext* cc) {
absl::Status ShotBoundaryCalculator::Open(mediapipe::CalculatorContext* cc) {
options_ = cc->Options<ShotBoundaryCalculatorOptions>();
last_shot_timestamp_ = Timestamp(0);
init_ = false;
return mediapipe::OkStatus();
return absl::OkStatus();
}
void ShotBoundaryCalculator::Transmit(mediapipe::CalculatorContext* cc,
@@ -127,8 +126,7 @@ void ShotBoundaryCalculator::Transmit(mediapipe::CalculatorContext* cc,
}
}
mediapipe::Status ShotBoundaryCalculator::Process(
mediapipe::CalculatorContext* cc) {
absl::Status ShotBoundaryCalculator::Process(mediapipe::CalculatorContext* cc) {
// Connect to input frame and make a mutable copy.
cv::Mat frame_org = mediapipe::formats::MatView(
&cc->Inputs().Tag(kVideoInputTag).Get<ImageFrame>());
@@ -142,7 +140,7 @@ mediapipe::Status ShotBoundaryCalculator::Process(
last_histogram_ = current_histogram;
init_ = true;
Transmit(cc, false);
return mediapipe::OkStatus();
return absl::OkStatus();
}
double current_motion_estimate =
@@ -152,7 +150,7 @@ mediapipe::Status ShotBoundaryCalculator::Process(
if (motion_history_.size() != options_.window_size()) {
Transmit(cc, false);
return mediapipe::OkStatus();
return absl::OkStatus();
}
// Shot detection algorithm is a mixture of adaptive (controlled with
@@ -176,14 +174,14 @@ mediapipe::Status ShotBoundaryCalculator::Process(
// Store histogram for next frame.
last_histogram_ = current_histogram;
motion_history_.pop_back();
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status ShotBoundaryCalculator::GetContract(
absl::Status ShotBoundaryCalculator::GetContract(
mediapipe::CalculatorContract* cc) {
cc->Inputs().Tag(kVideoInputTag).Set<ImageFrame>();
cc->Outputs().Tag(kShotChangeTag).Set<bool>();
return mediapipe::OkStatus();
return absl::OkStatus();
}
} // namespace autoflip
mediapipe/examples/desktop/autoflip/calculators/shot_boundary_calculator_test.cc
+1
View File
@@ -19,6 +19,7 @@
#include "mediapipe/framework/deps/file_path.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/formats/image_frame_opencv.h"
#include "mediapipe/framework/port/commandlineflags.h"
#include "mediapipe/framework/port/gmock.h"
#include "mediapipe/framework/port/gtest.h"
#include "mediapipe/framework/port/opencv_core_inc.h"
mediapipe/examples/desktop/autoflip/calculators/signal_fusing_calculator.cc
+15 -18
View File
@@ -105,13 +105,13 @@ class SignalFusingCalculator : public mediapipe::CalculatorBase {
SignalFusingCalculator(const SignalFusingCalculator&) = delete;
SignalFusingCalculator& operator=(const SignalFusingCalculator&) = delete;
static mediapipe::Status GetContract(mediapipe::CalculatorContract* cc);
mediapipe::Status Open(mediapipe::CalculatorContext* cc) override;
mediapipe::Status Process(mediapipe::CalculatorContext* cc) override;
mediapipe::Status Close(mediapipe::CalculatorContext* cc) override;
static absl::Status GetContract(mediapipe::CalculatorContract* cc);
absl::Status Open(mediapipe::CalculatorContext* cc) override;
absl::Status Process(mediapipe::CalculatorContext* cc) override;
absl::Status Close(mediapipe::CalculatorContext* cc) override;
private:
mediapipe::Status ProcessScene(mediapipe::CalculatorContext* cc);
absl::Status ProcessScene(mediapipe::CalculatorContext* cc);
std::vector<Packet> GetSignalPackets(mediapipe::CalculatorContext* cc);
SignalFusingCalculatorOptions options_;
std::map<std::string, SignalSettings> settings_by_type_;
@@ -154,8 +154,7 @@ void SetupOrderedInput(mediapipe::CalculatorContract* cc) {
}
} // namespace
mediapipe::Status SignalFusingCalculator::Open(
mediapipe::CalculatorContext* cc) {
absl::Status SignalFusingCalculator::Open(mediapipe::CalculatorContext* cc) {
options_ = cc->Options<SignalFusingCalculatorOptions>();
for (const auto& setting : options_.signal_settings()) {
settings_by_type_[CreateSettingsKey(setting.type())] = setting;
@@ -166,19 +165,18 @@ mediapipe::Status SignalFusingCalculator::Open(
process_by_scene_ = false;
}
}
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status SignalFusingCalculator::Close(
mediapipe::CalculatorContext* cc) {
absl::Status SignalFusingCalculator::Close(mediapipe::CalculatorContext* cc) {
if (!scene_frames_.empty()) {
MP_RETURN_IF_ERROR(ProcessScene(cc));
scene_frames_.clear();
}
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status SignalFusingCalculator::ProcessScene(
absl::Status SignalFusingCalculator::ProcessScene(
mediapipe::CalculatorContext* cc) {
std::map<std::string, int> detection_count;
std::map<std::string, float> multiframe_score;
@@ -240,7 +238,7 @@ mediapipe::Status SignalFusingCalculator::ProcessScene(
}
}
return mediapipe::OkStatus();
return absl::OkStatus();
}
std::vector<Packet> SignalFusingCalculator::GetSignalPackets(
@@ -260,8 +258,7 @@ std::vector<Packet> SignalFusingCalculator::GetSignalPackets(
return signal_packets;
}
mediapipe::Status SignalFusingCalculator::Process(
mediapipe::CalculatorContext* cc) {
absl::Status SignalFusingCalculator::Process(mediapipe::CalculatorContext* cc) {
bool is_boundary = false;
if (process_by_scene_) {
const auto& shot_tag = (tag_input_interface_)
@@ -302,17 +299,17 @@ mediapipe::Status SignalFusingCalculator::Process(
scene_frames_.clear();
}
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status SignalFusingCalculator::GetContract(
absl::Status SignalFusingCalculator::GetContract(
mediapipe::CalculatorContract* cc) {
if (cc->Inputs().NumEntries(kSignalInputsTag) > 0) {
SetupTagInput(cc);
} else {
SetupOrderedInput(cc);
}
return mediapipe::OkStatus();
return absl::OkStatus();
}
} // namespace autoflip
mediapipe/examples/desktop/autoflip/calculators/video_filtering_calculator.cc
+8 -9
View File
@@ -57,20 +57,19 @@ class VideoFilteringCalculator : public CalculatorBase {
VideoFilteringCalculator() = default;
~VideoFilteringCalculator() override = default;
static mediapipe::Status GetContract(CalculatorContract* cc);
static absl::Status GetContract(CalculatorContract* cc);
mediapipe::Status Process(CalculatorContext* cc) override;
absl::Status Process(CalculatorContext* cc) override;
};
REGISTER_CALCULATOR(VideoFilteringCalculator);
mediapipe::Status VideoFilteringCalculator::GetContract(
CalculatorContract* cc) {
absl::Status VideoFilteringCalculator::GetContract(CalculatorContract* cc) {
cc->Inputs().Tag(kInputFrameTag).Set<ImageFrame>();
cc->Outputs().Tag(kOutputFrameTag).Set<ImageFrame>();
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status VideoFilteringCalculator::Process(CalculatorContext* cc) {
absl::Status VideoFilteringCalculator::Process(CalculatorContext* cc) {
const auto& options = cc->Options<VideoFilteringCalculatorOptions>();
const Packet& input_packet = cc->Inputs().Tag(kInputFrameTag).Value();
@@ -84,7 +83,7 @@ mediapipe::Status VideoFilteringCalculator::Process(CalculatorContext* cc) {
if (filter_type ==
VideoFilteringCalculatorOptions::AspectRatioFilter::NO_FILTERING) {
cc->Outputs().Tag(kOutputFrameTag).AddPacket(input_packet);
return mediapipe::OkStatus();
return absl::OkStatus();
}
const int target_width = options.aspect_ratio_filter().target_width();
const int target_height = options.aspect_ratio_filter().target_height();
@@ -106,7 +105,7 @@ mediapipe::Status VideoFilteringCalculator::Process(CalculatorContext* cc) {
}
if (should_pass) {
cc->Outputs().Tag(kOutputFrameTag).AddPacket(input_packet);
return mediapipe::OkStatus();
return absl::OkStatus();
}
if (options.fail_if_any()) {
return mediapipe::UnknownErrorBuilder(MEDIAPIPE_LOC) << absl::Substitute(
@@ -115,7 +114,7 @@ mediapipe::Status VideoFilteringCalculator::Process(CalculatorContext* cc) {
target_ratio, frame.Width(), frame.Height());
}
return mediapipe::OkStatus();
return absl::OkStatus();
}
} // namespace autoflip
} // namespace mediapipe
mediapipe/examples/desktop/autoflip/calculators/video_filtering_calculator_test.cc
+2 -2
View File
@@ -166,8 +166,8 @@ TEST(VerticalFrameRemovalCalculatorTest, OutputError) {
runner->MutableInputs()
->Tag("INPUT_FRAMES")
.packets.push_back(Adopt(input_frame.release()).At(Timestamp(1000)));
mediapipe::Status status = runner->Run();
EXPECT_EQ(status.code(), mediapipe::StatusCode::kUnknown);
absl::Status status = runner->Run();
EXPECT_EQ(status.code(), absl::StatusCode::kUnknown);
EXPECT_THAT(status.ToString(),
::testing::HasSubstr("Failing due to aspect ratio"));
}
mediapipe/examples/desktop/autoflip/quality/BUILD
+1
View File
@@ -249,6 +249,7 @@ cc_test(
":scene_camera_motion_analyzer",
"//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto",
"//mediapipe/framework/deps:file_path",
"//mediapipe/framework/port:commandlineflags",
"//mediapipe/framework/port:file_helpers",
"//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:status",
mediapipe/examples/desktop/autoflip/quality/frame_crop_region_computer.cc
+6 -6
View File
@@ -22,7 +22,7 @@
namespace mediapipe {
namespace autoflip {
mediapipe::Status FrameCropRegionComputer::ExpandSegmentUnderConstraint(
absl::Status FrameCropRegionComputer::ExpandSegmentUnderConstraint(
const Segment& segment_to_add, const Segment& base_segment,
const int max_length, Segment* combined_segment,
CoverType* cover_type) const {
@@ -75,10 +75,10 @@ mediapipe::Status FrameCropRegionComputer::ExpandSegmentUnderConstraint(
*combined_segment =
std::make_pair(combined_segment_left, combined_segment_right);
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status FrameCropRegionComputer::ExpandRectUnderConstraints(
absl::Status FrameCropRegionComputer::ExpandRectUnderConstraints(
const Rect& rect_to_add, const int max_width, const int max_height,
Rect* base_rect, CoverType* cover_type) const {
RET_CHECK(base_rect != nullptr) << "Base rect is null.";
@@ -129,7 +129,7 @@ mediapipe::Status FrameCropRegionComputer::ExpandRectUnderConstraints(
}
}
return mediapipe::OkStatus();
return absl::OkStatus();
}
void FrameCropRegionComputer::UpdateCropRegionScore(
@@ -167,7 +167,7 @@ void FrameCropRegionComputer::UpdateCropRegionScore(
}
}
mediapipe::Status FrameCropRegionComputer::ComputeFrameCropRegion(
absl::Status FrameCropRegionComputer::ComputeFrameCropRegion(
const KeyFrameInfo& frame_info, KeyFrameCropResult* crop_result) const {
RET_CHECK(crop_result != nullptr) << "KeyFrameCropResult is null.";
@@ -254,7 +254,7 @@ mediapipe::Status FrameCropRegionComputer::ComputeFrameCropRegion(
crop_result->set_region_is_empty(crop_region_is_empty);
crop_result->set_region_score(crop_region_score);
return mediapipe::OkStatus();
return absl::OkStatus();
}
} // namespace autoflip
mediapipe/examples/desktop/autoflip/quality/frame_crop_region_computer.h
+11 -12
View File
@@ -43,8 +43,8 @@ class FrameCropRegionComputer {
// consider static features, and simply tries to fit the detected features
// within the target frame size. The score of the crop region is aggregated
// from individual feature scores given the score aggregation type.
mediapipe::Status ComputeFrameCropRegion(
const KeyFrameInfo& frame_info, KeyFrameCropResult* crop_result) const;
absl::Status ComputeFrameCropRegion(const KeyFrameInfo& frame_info,
KeyFrameCropResult* crop_result) const;
protected:
// A segment is a 1-d object defined by its left and right point.
@@ -75,11 +75,11 @@ class FrameCropRegionComputer {
// fraction of the new segment exceeds the maximum length.
// In this case the combined segment is the base segment, and cover
// type is NOT_COVERED.
mediapipe::Status ExpandSegmentUnderConstraint(const Segment& segment_to_add,
const Segment& base_segment,
const int max_length,
Segment* combined_segment,
CoverType* cover_type) const;
absl::Status ExpandSegmentUnderConstraint(const Segment& segment_to_add,
const Segment& base_segment,
const int max_length,
Segment* combined_segment,
CoverType* cover_type) const;
// Expands a base rectangle to cover a new rectangle to be added under width
// and height constraints. The operation is best-effort. It considers
@@ -88,11 +88,10 @@ class FrameCropRegionComputer {
// FULLY_COVERED if the new rectangle is fully covered in both directions,
// PARTIALLY_COVERED if it is at least partially covered in both directions,
// and NOT_COVERED if it is not covered in either direction.
mediapipe::Status ExpandRectUnderConstraints(const Rect& rect_to_add,
const int max_width,
const int max_height,
Rect* base_rect,
CoverType* cover_type) const;
absl::Status ExpandRectUnderConstraints(const Rect& rect_to_add,
const int max_width,
const int max_height, Rect* base_rect,
CoverType* cover_type) const;
// Updates crop region score given current feature score, whether the feature
// is required, and the score aggregation type. Ignores negative scores.
mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver.cc
+107 -15
View File
@@ -14,10 +14,66 @@ int Median(const std::deque<std::pair<uint64, int>>& positions_raw) {
return positions[n];
}
} // namespace
mediapipe::Status KinematicPathSolver::AddObservation(int position,
const uint64 time_us) {
bool KinematicPathSolver::IsMotionTooSmall(double delta_degs) {
if (options_.has_min_motion_to_reframe()) {
return abs(delta_degs) < options_.min_motion_to_reframe();
} else if (delta_degs > 0) {
return delta_degs < options_.min_motion_to_reframe_upper();
} else {
return abs(delta_degs) < options_.min_motion_to_reframe_lower();
}
}
void KinematicPathSolver::ClearHistory() { raw_positions_at_time_.clear(); }
absl::Status KinematicPathSolver::PredictMotionState(int position,
const uint64 time_us,
bool* state) {
if (!initialized_) {
current_position_px_ = position;
*state = false;
return absl::OkStatus();
}
auto raw_positions_at_time_copy = raw_positions_at_time_;
raw_positions_at_time_copy.push_front(
std::pair<uint64, int>(time_us, position));
while (raw_positions_at_time_copy.size() > 1) {
if (static_cast<int64>(raw_positions_at_time_copy.back().first) <
static_cast<int64>(time_us) - options_.filtering_time_window_us()) {
raw_positions_at_time_copy.pop_back();
} else {
break;
}
}
int filtered_position = Median(raw_positions_at_time_copy);
double delta_degs =
(filtered_position - current_position_px_) / pixels_per_degree_;
// If the motion is smaller than the min_motion_to_reframe and camera is
// stationary, don't use the update.
if (IsMotionTooSmall(delta_degs) && !motion_state_) {
*state = false;
} else if (abs(delta_degs) < options_.reframe_window() && motion_state_) {
// If the motion is smaller than the reframe_window and camera is moving,
// don't use the update.
*state = false;
} else {
// Apply new position, plus the reframe window size.
*state = true;
}
return absl::OkStatus();
}
absl::Status KinematicPathSolver::AddObservation(int position,
const uint64 time_us) {
if (!initialized_) {
if (position < min_location_) {
current_position_px_ = min_location_;
} else if (position > max_location_) {
current_position_px_ = max_location_;
} else {
current_position_px_ = position;
}
target_position_px_ = position;
motion_state_ = false;
mean_delta_t_ = -1;
@@ -30,13 +86,27 @@ mediapipe::Status KinematicPathSolver::AddObservation(int position,
<< "pixels_per_degree must be larger than 0.";
RET_CHECK_GE(options_.update_rate_seconds(), 0)
<< "update_rate_seconds must be greater than 0.";
RET_CHECK_GE(options_.min_motion_to_reframe(), options_.reframe_window())
<< "Reframe window cannot exceed min_motion_to_reframe.";
RET_CHECK_GE(options_.filtering_time_window_us(), 0)
<< "update_rate_seconds must be greater than 0.";
RET_CHECK_GE(options_.mean_period_update_rate(), 0)
<< "mean_period_update_rate must be greater than 0.";
return mediapipe::OkStatus();
RET_CHECK(options_.has_min_motion_to_reframe() ^
(options_.has_min_motion_to_reframe_upper() &&
options_.has_min_motion_to_reframe_lower()))
<< "Must set min_motion_to_reframe or min_motion_to_reframe_upper and "
"min_motion_to_reframe_lower.";
if (options_.has_min_motion_to_reframe()) {
RET_CHECK_GE(options_.min_motion_to_reframe(), options_.reframe_window())
<< "Reframe window cannot exceed min_motion_to_reframe.";
} else {
RET_CHECK_GE(options_.min_motion_to_reframe_upper(),
options_.reframe_window())
<< "Reframe window cannot exceed min_motion_to_reframe.";
RET_CHECK_GE(options_.min_motion_to_reframe_lower(),
options_.reframe_window())
<< "Reframe window cannot exceed min_motion_to_reframe.";
}
return absl::OkStatus();
}
RET_CHECK(current_time_ < time_us)
@@ -58,7 +128,7 @@ mediapipe::Status KinematicPathSolver::AddObservation(int position,
// If the motion is smaller than the min_motion_to_reframe and camera is
// stationary, don't use the update.
if (abs(delta_degs) < options_.min_motion_to_reframe() && !motion_state_) {
if (IsMotionTooSmall(delta_degs) && !motion_state_) {
delta_degs = 0;
motion_state_ = false;
} else if (abs(delta_degs) < options_.reframe_window() && motion_state_) {
@@ -100,7 +170,7 @@ mediapipe::Status KinematicPathSolver::AddObservation(int position,
return UpdatePrediction(time_us);
}
mediapipe::Status KinematicPathSolver::UpdatePrediction(const int64 time_us) {
absl::Status KinematicPathSolver::UpdatePrediction(const int64 time_us) {
RET_CHECK(current_time_ < time_us)
<< "Prediction time added before a prior observation or prediction.";
@@ -122,36 +192,58 @@ mediapipe::Status KinematicPathSolver::UpdatePrediction(const int64 time_us) {
if (update_position_px < min_location_) {
current_position_px_ = min_location_;
current_velocity_deg_per_s_ = 0;
motion_state_ = false;
} else if (update_position_px > max_location_) {
current_position_px_ = max_location_;
current_velocity_deg_per_s_ = 0;
motion_state_ = false;
} else {
current_position_px_ = update_position_px;
}
current_time_ = time_us;
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status KinematicPathSolver::GetState(int* position) {
absl::Status KinematicPathSolver::GetState(int* position) {
RET_CHECK(initialized_) << "GetState called before first observation added.";
*position = round(current_position_px_);
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status KinematicPathSolver::GetTargetPosition(int* target_position) {
absl::Status KinematicPathSolver::GetTargetPosition(int* target_position) {
RET_CHECK(initialized_)
<< "GetTargetPosition called before first observation added.";
*target_position = round(target_position_px_);
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status KinematicPathSolver::UpdatePixelsPerDegree(
absl::Status KinematicPathSolver::UpdatePixelsPerDegree(
const float pixels_per_degree) {
RET_CHECK_GT(pixels_per_degree_, 0)
<< "pixels_per_degree must be larger than 0.";
pixels_per_degree_ = pixels_per_degree;
return mediapipe::OkStatus();
return absl::OkStatus();
}
absl::Status KinematicPathSolver::UpdateMinMaxLocation(const int min_location,
const int max_location) {
RET_CHECK(initialized_)
<< "UpdateMinMaxLocation called before first observation added.";
double prior_distance = max_location_ - min_location_;
double updated_distance = max_location - min_location;
double scale_change = updated_distance / prior_distance;
current_position_px_ = current_position_px_ * scale_change;
target_position_px_ = target_position_px_ * scale_change;
max_location_ = max_location;
min_location_ = min_location;
auto original_positions_at_time = raw_positions_at_time_;
raw_positions_at_time_.clear();
for (auto position_at_time : original_positions_at_time) {
position_at_time.second = position_at_time.second * scale_change;
raw_positions_at_time_.push_front(position_at_time);
}
return absl::OkStatus();
}
} // namespace autoflip
mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver.h
+19 -7
View File
@@ -43,22 +43,34 @@ class KinematicPathSolver {
initialized_(false),
pixels_per_degree_(pixels_per_degree) {}
// Add an observation (detection) at a position and time.
mediapipe::Status AddObservation(int position, const uint64 time_us);
absl::Status AddObservation(int position, const uint64 time_us);
// Get the predicted position at a time.
mediapipe::Status UpdatePrediction(const int64 time_us);
absl::Status UpdatePrediction(const int64 time_us);
// Get the state at a time.
mediapipe::Status GetState(int* position);
absl::Status GetState(int* position);
// Update PixelPerDegree value.
mediapipe::Status UpdatePixelsPerDegree(const float pixels_per_degree);
absl::Status UpdatePixelsPerDegree(const float pixels_per_degree);
// Provide the current target position of the reframe action.
mediapipe::Status GetTargetPosition(int* target_position);
absl::Status GetTargetPosition(int* target_position);
// Change min/max location and update state based on new scaling.
absl::Status UpdateMinMaxLocation(const int min_location,
const int max_location);
// Check if motion is within the reframe window, return false if not.
bool IsMotionTooSmall(double delta_degs);
// Check if a position measurement will cause the camera to be in motion
// without updating the internal state.
absl::Status PredictMotionState(int position, const uint64 time_us,
bool* state);
// Clear any history buffer of positions that are used when
// filtering_time_window_us is set to a non-zero value.
void ClearHistory();
private:
// Tuning options.
KinematicOptions options_;
// Min and max value the state can be.
const int min_location_;
const int max_location_;
int min_location_;
int max_location_;
bool initialized_;
float pixels_per_degree_;
// Current state values.
mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver.proto
+8 -2
View File
@@ -8,8 +8,14 @@ message KinematicOptions {
optional double update_rate = 1 [default = 0.5, deprecated = true];
// Max velocity (degrees per second) that the camera can move.
optional double max_velocity = 2 [default = 18];
// Min motion (in degrees) to react in pixels.
optional float min_motion_to_reframe = 3 [default = 1.8];
// Min motion (in degrees) to react for both upper and lower directions. Must
// not be set if using min_motion_to_reframe_lower and
// min_motion_to_reframe_upper.
optional float min_motion_to_reframe = 3;
// Min motion (in degrees) for upper and lower direction to react. Both must
// be set and min_motion_to_reframe cannot be set if these are specified.
optional float min_motion_to_reframe_lower = 9;
optional float min_motion_to_reframe_upper = 10;
// When motion exceeds min_motion_to_reframe, move within this distance of the
// camera from the starting direction. Setting this value non-zero reduces
// total reframe distance on average. Value cannot exceed
mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver_test.cc
+64
View File
@@ -190,6 +190,50 @@ TEST(KinematicPathSolverTest, PassReframeWindow) {
EXPECT_EQ(state, 508);
}
TEST(KinematicPathSolverTest, PassReframeWindowLowerUpper) {
KinematicOptions options;
// Set min motion to 1deg
options.set_min_motion_to_reframe_upper(1.3);
options.set_min_motion_to_reframe_lower(1.0);
options.set_update_rate_seconds(.0000001);
options.set_max_update_rate(1.0);
options.set_max_velocity(1000);
// Set reframe window size to .75 for test.
options.set_reframe_window(0.75);
// Set degrees / pixel to 16.6
KinematicPathSolver solver(options, 0, 1000, 1000.0 / kWidthFieldOfView);
int state;
MP_ASSERT_OK(solver.AddObservation(500, kMicroSecInSec * 0));
// Move target by 20px / 16.6 = 1.2deg
MP_ASSERT_OK(solver.AddObservation(520, kMicroSecInSec * 1));
MP_ASSERT_OK(solver.GetState(&state));
// Expect cam to not move
EXPECT_EQ(state, 500);
MP_ASSERT_OK(solver.AddObservation(480, kMicroSecInSec * 2));
MP_ASSERT_OK(solver.GetState(&state));
// Expect cam to move
EXPECT_EQ(state, 493);
}
TEST(KinematicPathSolverTest, PassCheckState) {
KinematicOptions options;
// Set min motion to 1deg
options.set_min_motion_to_reframe(1.0);
options.set_update_rate_seconds(.0000001);
options.set_max_update_rate(1.0);
options.set_max_velocity(1000);
// Set reframe window size to .75 for test.
options.set_reframe_window(0.75);
// Set degrees / pixel to 16.6
KinematicPathSolver solver(options, 0, 1000, 1000.0 / kWidthFieldOfView);
MP_ASSERT_OK(solver.AddObservation(500, kMicroSecInSec * 0));
// Move target by 20px / 16.6 = 1.2deg
bool motion_state;
MP_ASSERT_OK(
solver.PredictMotionState(520, kMicroSecInSec * 1, &motion_state));
EXPECT_TRUE(motion_state);
}
TEST(KinematicPathSolverTest, PassUpdateRate30FPS) {
KinematicOptions options;
options.set_min_motion_to_reframe(1.0);
@@ -238,6 +282,26 @@ TEST(KinematicPathSolverTest, PassUpdateRate) {
EXPECT_EQ(state, 505);
}
TEST(KinematicPathSolverTest, PassUpdateRateResolutionChange) {
KinematicOptions options;
options.set_min_motion_to_reframe(1.0);
options.set_update_rate_seconds(4);
options.set_max_update_rate(1.0);
options.set_max_velocity(18);
KinematicPathSolver solver(options, 0, 1000, 1000.0 / kWidthFieldOfView);
int state, target_position;
MP_ASSERT_OK(solver.AddObservation(500, kMicroSecInSec * 0));
MP_ASSERT_OK(solver.GetTargetPosition(&target_position));
EXPECT_EQ(target_position, 500);
MP_ASSERT_OK(solver.UpdateMinMaxLocation(0, 500));
MP_ASSERT_OK(solver.UpdatePixelsPerDegree(500.0 / kWidthFieldOfView));
MP_ASSERT_OK(solver.AddObservation(520 * 0.5, kMicroSecInSec * 1));
MP_ASSERT_OK(solver.GetTargetPosition(&target_position));
EXPECT_EQ(target_position, 520 * 0.5);
MP_ASSERT_OK(solver.GetState(&state));
EXPECT_EQ(state, 253);
}
TEST(KinematicPathSolverTest, PassMaxVelocity) {
KinematicOptions options;
options.set_min_motion_to_reframe(1.0);
mediapipe/examples/desktop/autoflip/quality/padding_effect_generator.cc
+2 -2
View File
@@ -45,7 +45,7 @@ PaddingEffectGenerator::PaddingEffectGenerator(const int input_width,
}
}
mediapipe::Status PaddingEffectGenerator::Process(
absl::Status PaddingEffectGenerator::Process(
const ImageFrame& input_frame, const float background_contrast,
const int blur_cv_size, const float overlay_opacity,
ImageFrame* output_frame, const cv::Scalar* background_color_in_rgb) {
@@ -170,7 +170,7 @@ mediapipe::Status PaddingEffectGenerator::Process(
output_frame->CopyPixelData(input_frame.Format(), canvas.cols, canvas.rows,
canvas.data,
ImageFrame::kDefaultAlignmentBoundary);
return mediapipe::OkStatus();
return absl::OkStatus();
}
cv::Rect PaddingEffectGenerator::ComputeOutputLocation() {
mediapipe/examples/desktop/autoflip/quality/padding_effect_generator.h
+4 -5
View File
@@ -49,11 +49,10 @@ class PaddingEffectGenerator {
// the opacity of the black layer.
// - background_color_in_rgb: If not null, uses this solid color as background
// instead of blurring the image, and does not adjust contrast or opacity.
mediapipe::Status Process(
const ImageFrame& input_frame, const float background_contrast,
const int blur_cv_size, const float overlay_opacity,
ImageFrame* output_frame,
const cv::Scalar* background_color_in_rgb = nullptr);
absl::Status Process(const ImageFrame& input_frame,
const float background_contrast, const int blur_cv_size,
const float overlay_opacity, ImageFrame* output_frame,
const cv::Scalar* background_color_in_rgb = nullptr);
// Compute the "render location" on the output frame where the "crop from"
// location is to be placed. For use with external rendering soutions.
mediapipe/examples/desktop/autoflip/quality/padding_effect_generator_test.cc
+5 -3
View File
@@ -48,12 +48,14 @@ const cv::Scalar kRed = cv::Scalar(255, 0, 0);
void TestWithAspectRatio(const double aspect_ratio,
const cv::Scalar* background_color_in_rgb = nullptr) {
std::string test_image;
const bool process_arbitrary_image = !FLAGS_input_image.empty();
const bool process_arbitrary_image =
!absl::GetFlag(FLAGS_input_image).empty();
if (!process_arbitrary_image) {
std::string test_image_path = mediapipe::file::JoinPath("./", kTestImage);
MP_ASSERT_OK(mediapipe::file::GetContents(test_image_path, &test_image));
} else {
MP_ASSERT_OK(mediapipe::file::GetContents(FLAGS_input_image, &test_image));
MP_ASSERT_OK(mediapipe::file::GetContents(absl::GetFlag(FLAGS_input_image),
&test_image));
}
const std::vector<char> contents_vector(test_image.begin(), test_image.end());
@@ -138,7 +140,7 @@ void TestWithAspectRatio(const double aspect_ratio,
EXPECT_EQ(result_image, output_string);
} else {
std::string output_string_path = mediapipe::file::JoinPath(
FLAGS_output_folder,
absl::GetFlag(FLAGS_output_folder),
absl::StrCat("result_", aspect_ratio,
background_color_in_rgb ? "_solid_background" : "",
".jpg"));
mediapipe/examples/desktop/autoflip/quality/polynomial_regression_path_solver.cc
+2 -2
View File
@@ -91,7 +91,7 @@ void PolynomialRegressionPathSolver::AddCostFunctionToProblem(
problem->AddResidualBlock(cost_function, new CauchyLoss(0.5), a, b, c, d, k);
}
mediapipe::Status PolynomialRegressionPathSolver::ComputeCameraPath(
absl::Status PolynomialRegressionPathSolver::ComputeCameraPath(
const std::vector<FocusPointFrame>& focus_point_frames,
const std::vector<FocusPointFrame>& prior_focus_point_frames,
const int original_width, const int original_height, const int output_width,
@@ -163,7 +163,7 @@ mediapipe::Status PolynomialRegressionPathSolver::ComputeCameraPath(
}
all_transforms->push_back(transform);
}
return mediapipe::OkStatus();
return absl::OkStatus();
}
} // namespace autoflip
mediapipe/examples/desktop/autoflip/quality/polynomial_regression_path_solver.h
+1 -1
View File
@@ -42,7 +42,7 @@ class PolynomialRegressionPathSolver {
// y-axis, such that focus points can be preserved as much as possible. The
// returned |all_transforms| hold the camera location at each timestamp
// corresponding to each input frame.
mediapipe::Status ComputeCameraPath(
absl::Status ComputeCameraPath(
const std::vector<FocusPointFrame>& focus_point_frames,
const std::vector<FocusPointFrame>& prior_focus_point_frames,
const int original_width, const int original_height,
mediapipe/examples/desktop/autoflip/quality/scene_camera_motion_analyzer.cc
+19 -23
View File
@@ -30,8 +30,7 @@
namespace mediapipe {
namespace autoflip {
mediapipe::Status
SceneCameraMotionAnalyzer::AnalyzeSceneAndPopulateFocusPointFrames(
absl::Status SceneCameraMotionAnalyzer::AnalyzeSceneAndPopulateFocusPointFrames(
const KeyFrameCropOptions& key_frame_crop_options,
const std::vector<KeyFrameCropResult>& key_frame_crop_results,
const int scene_frame_width, const int scene_frame_height,
@@ -67,7 +66,7 @@ SceneCameraMotionAnalyzer::AnalyzeSceneAndPopulateFocusPointFrames(
scene_frame_timestamps, focus_point_frames);
}
mediapipe::Status SceneCameraMotionAnalyzer::ToUseSteadyMotion(
absl::Status SceneCameraMotionAnalyzer::ToUseSteadyMotion(
const float look_at_center_x, const float look_at_center_y,
const int crop_window_width, const int crop_window_height,
SceneKeyFrameCropSummary* scene_summary,
@@ -77,10 +76,10 @@ mediapipe::Status SceneCameraMotionAnalyzer::ToUseSteadyMotion(
auto* steady_motion = scene_camera_motion->mutable_steady_motion();
steady_motion->set_steady_look_at_center_x(look_at_center_x);
steady_motion->set_steady_look_at_center_y(look_at_center_y);
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status SceneCameraMotionAnalyzer::ToUseSweepingMotion(
absl::Status SceneCameraMotionAnalyzer::ToUseSweepingMotion(
const float start_x, const float start_y, const float end_x,
const float end_y, const int crop_window_width,
const int crop_window_height, const double time_duration_in_sec,
@@ -99,10 +98,10 @@ mediapipe::Status SceneCameraMotionAnalyzer::ToUseSweepingMotion(
scene_summary->frame_success_rate(), start_x, start_y, end_x, end_y,
time_duration_in_sec);
VLOG(1) << sweeping_log;
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status SceneCameraMotionAnalyzer::DecideCameraMotionType(
absl::Status SceneCameraMotionAnalyzer::DecideCameraMotionType(
const KeyFrameCropOptions& key_frame_crop_options,
const double scene_span_sec, const int64 end_time_us,
SceneKeyFrameCropSummary* scene_summary,
@@ -131,7 +130,7 @@ mediapipe::Status SceneCameraMotionAnalyzer::DecideCameraMotionType(
no_salient_position_x, no_salient_position_y,
scene_summary->crop_window_width(), scene_summary->crop_window_height(),
scene_summary, scene_camera_motion));
return mediapipe::OkStatus();
return absl::OkStatus();
}
// Sweep across the scene when 1) success rate is too low, AND 2) the current
@@ -164,7 +163,7 @@ mediapipe::Status SceneCameraMotionAnalyzer::DecideCameraMotionType(
start_x, start_y, end_x, end_y, key_frame_crop_options.target_width(),
key_frame_crop_options.target_height(), scene_span_sec, scene_summary,
scene_camera_motion));
return mediapipe::OkStatus();
return absl::OkStatus();
}
// If scene motion is small, then look at a steady point in the scene.
@@ -179,14 +178,14 @@ mediapipe::Status SceneCameraMotionAnalyzer::DecideCameraMotionType(
// Otherwise, tracks the focus regions.
scene_camera_motion->mutable_tracking_motion();
return mediapipe::OkStatus();
return absl::OkStatus();
}
// If there is no required focus region, looks at the middle of the center
// range, and snaps to the scene center if close. Otherwise, look at the center
// of the union of the required focus regions, and ensures the crop region
// covers this union.
mediapipe::Status SceneCameraMotionAnalyzer::DecideSteadyLookAtRegion(
absl::Status SceneCameraMotionAnalyzer::DecideSteadyLookAtRegion(
const KeyFrameCropOptions& key_frame_crop_options,
SceneKeyFrameCropSummary* scene_summary,
SceneCameraMotion* scene_camera_motion) const {
@@ -252,11 +251,10 @@ mediapipe::Status SceneCameraMotionAnalyzer::DecideSteadyLookAtRegion(
MP_RETURN_IF_ERROR(ToUseSteadyMotion(center_x, center_y, crop_width,
crop_height, scene_summary,
scene_camera_motion));
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status
SceneCameraMotionAnalyzer::AddFocusPointsFromCenterTypeAndWeight(
absl::Status SceneCameraMotionAnalyzer::AddFocusPointsFromCenterTypeAndWeight(
const float center_x, const float center_y, const int frame_width,
const int frame_height, const FocusPointFrameType type, const float weight,
const float bound, FocusPointFrame* focus_point_frame) const {
@@ -294,10 +292,10 @@ SceneCameraMotionAnalyzer::AddFocusPointsFromCenterTypeAndWeight(
} else {
RET_CHECK_FAIL() << absl::StrCat("Invalid FocusPointFrameType ", type);
}
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status SceneCameraMotionAnalyzer::PopulateFocusPointFrames(
absl::Status SceneCameraMotionAnalyzer::PopulateFocusPointFrames(
const SceneKeyFrameCropSummary& scene_summary,
const SceneCameraMotion& scene_camera_motion,
const std::vector<int64>& scene_frame_timestamps,
@@ -340,7 +338,7 @@ mediapipe::Status SceneCameraMotionAnalyzer::PopulateFocusPointFrames(
options_.salient_point_bound(), &focus_point_frame));
focus_point_frames->push_back(focus_point_frame);
}
return mediapipe::OkStatus();
return absl::OkStatus();
} else if (scene_camera_motion.has_sweeping_motion()) {
// Camera sweeps across the frame.
const auto& sweeping_motion = scene_camera_motion.sweeping_motion();
@@ -361,7 +359,7 @@ mediapipe::Status SceneCameraMotionAnalyzer::PopulateFocusPointFrames(
options_.salient_point_bound(), &focus_point_frame));
focus_point_frames->push_back(focus_point_frame);
}
return mediapipe::OkStatus();
return absl::OkStatus();
} else if (scene_camera_motion.has_tracking_motion()) {
// Camera tracks crop regions.
RET_CHECK_GT(scene_summary.num_key_frames(), 0) << "No key frames.";
@@ -369,8 +367,7 @@ mediapipe::Status SceneCameraMotionAnalyzer::PopulateFocusPointFrames(
scene_summary, focus_point_frame_type, scene_frame_timestamps,
focus_point_frames);
} else {
return mediapipe::Status(StatusCode::kInvalidArgument,
"Unknown motion type.");
return absl::Status(StatusCode::kInvalidArgument, "Unknown motion type.");
}
}
@@ -380,8 +377,7 @@ mediapipe::Status SceneCameraMotionAnalyzer::PopulateFocusPointFrames(
// The weight for the focus point is proportional to the interpolated score
// and scaled so that the maximum weight is equal to
// maximum_focus_point_weight in the SceneCameraMotionAnalyzerOptions.
mediapipe::Status
SceneCameraMotionAnalyzer::PopulateFocusPointFramesForTracking(
absl::Status SceneCameraMotionAnalyzer::PopulateFocusPointFramesForTracking(
const SceneKeyFrameCropSummary& scene_summary,
const FocusPointFrameType focus_point_frame_type,
const std::vector<int64>& scene_frame_timestamps,
@@ -440,7 +436,7 @@ SceneCameraMotionAnalyzer::PopulateFocusPointFramesForTracking(
focus_point->set_weight(scale * focus_point->weight());
}
}
return mediapipe::OkStatus();
return absl::OkStatus();
}
} // namespace autoflip
mediapipe/examples/desktop/autoflip/quality/scene_camera_motion_analyzer.h
+13 -12
View File
@@ -62,7 +62,7 @@ class SceneCameraMotionAnalyzer {
// Aggregates information from KeyFrameInfos and KeyFrameCropResults into
// SceneKeyFrameCropSummary, and populates FocusPointFrames given scene
// frame timestamps. Optionally returns SceneCameraMotion.
mediapipe::Status AnalyzeSceneAndPopulateFocusPointFrames(
absl::Status AnalyzeSceneAndPopulateFocusPointFrames(
const KeyFrameCropOptions& key_frame_crop_options,
const std::vector<KeyFrameCropResult>& key_frame_crop_results,
const int scene_frame_width, const int scene_frame_height,
@@ -75,7 +75,7 @@ class SceneCameraMotionAnalyzer {
protected:
// Decides SceneCameraMotion based on SceneKeyFrameCropSummary. Updates the
// crop window in SceneKeyFrameCropSummary in the case of steady motion.
mediapipe::Status DecideCameraMotionType(
absl::Status DecideCameraMotionType(
const KeyFrameCropOptions& key_frame_crop_options,
const double scene_span_sec, const int64 end_time_us,
SceneKeyFrameCropSummary* scene_summary,
@@ -83,7 +83,7 @@ class SceneCameraMotionAnalyzer {
// Populates the FocusPointFrames for each scene frame based on
// SceneKeyFrameCropSummary, SceneCameraMotion, and scene frame timestamps.
mediapipe::Status PopulateFocusPointFrames(
absl::Status PopulateFocusPointFrames(
const SceneKeyFrameCropSummary& scene_summary,
const SceneCameraMotion& scene_camera_motion,
const std::vector<int64>& scene_frame_timestamps,
@@ -91,7 +91,7 @@ class SceneCameraMotionAnalyzer {
private:
// Decides the look-at region when camera is steady.
mediapipe::Status DecideSteadyLookAtRegion(
absl::Status DecideSteadyLookAtRegion(
const KeyFrameCropOptions& key_frame_crop_options,
SceneKeyFrameCropSummary* scene_summary,
SceneCameraMotion* scene_camera_motion) const;
@@ -105,7 +105,7 @@ class SceneCameraMotionAnalyzer {
// Adds FocusPoint(s) to given FocusPointFrame given center location,
// frame size, FocusPointFrameType, weight, and bound.
mediapipe::Status AddFocusPointsFromCenterTypeAndWeight(
absl::Status AddFocusPointsFromCenterTypeAndWeight(
const float center_x, const float center_y, const int frame_width,
const int frame_height, const FocusPointFrameType type,
const float weight, const float bound,
@@ -114,21 +114,22 @@ class SceneCameraMotionAnalyzer {
// Populates the FocusPointFrames for each scene frame based on
// SceneKeyFrameCropSummary and scene frame timestamps in the case where
// camera is tracking the crop regions.
mediapipe::Status PopulateFocusPointFramesForTracking(
absl::Status PopulateFocusPointFramesForTracking(
const SceneKeyFrameCropSummary& scene_summary,
const FocusPointFrameType focus_point_frame_type,
const std::vector<int64>& scene_frame_timestamps,
std::vector<FocusPointFrame>* focus_point_frames) const;
// Decide to use steady motion.
mediapipe::Status ToUseSteadyMotion(
const float look_at_center_x, const float look_at_center_y,
const int crop_window_width, const int crop_window_height,
SceneKeyFrameCropSummary* scene_summary,
SceneCameraMotion* scene_camera_motion) const;
absl::Status ToUseSteadyMotion(const float look_at_center_x,
const float look_at_center_y,
const int crop_window_width,
const int crop_window_height,
SceneKeyFrameCropSummary* scene_summary,
SceneCameraMotion* scene_camera_motion) const;
// Decide to use sweeping motion.
mediapipe::Status ToUseSweepingMotion(
absl::Status ToUseSweepingMotion(
const float start_x, const float start_y, const float end_x,
const float end_y, const int crop_window_width,
const int crop_window_height, const double time_duration_in_sec,
mediapipe/examples/desktop/autoflip/quality/scene_camera_motion_analyzer_test.cc
+1
View File
@@ -24,6 +24,7 @@
#include "mediapipe/examples/desktop/autoflip/quality/focus_point.pb.h"
#include "mediapipe/examples/desktop/autoflip/quality/piecewise_linear_function.h"
#include "mediapipe/framework/deps/file_path.h"
#include "mediapipe/framework/port/commandlineflags.h"
#include "mediapipe/framework/port/file_helpers.h"
#include "mediapipe/framework/port/gmock.h"
#include "mediapipe/framework/port/gtest.h"
mediapipe/examples/desktop/autoflip/quality/scene_cropper.cc
+4 -4
View File
@@ -29,7 +29,7 @@ constexpr float kWidthFieldOfView = 60;
namespace mediapipe {
namespace autoflip {
mediapipe::Status SceneCropper::ProcessKinematicPathSolver(
absl::Status SceneCropper::ProcessKinematicPathSolver(
const SceneKeyFrameCropSummary& scene_summary,
const std::vector<int64>& scene_timestamps,
const std::vector<bool>& is_key_frames,
@@ -77,10 +77,10 @@ mediapipe::Status SceneCropper::ProcessKinematicPathSolver(
-(x_path - scene_summary.crop_window_width() / 2);
all_xforms->push_back(transform);
}
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status SceneCropper::CropFrames(
absl::Status SceneCropper::CropFrames(
const SceneKeyFrameCropSummary& scene_summary,
const std::vector<int64>& scene_timestamps,
const std::vector<bool>& is_key_frames,
@@ -151,7 +151,7 @@ mediapipe::Status SceneCropper::CropFrames(
// If no cropped_frames is passed in, return directly.
if (!cropped_frames) {
return mediapipe::OkStatus();
return absl::OkStatus();
}
RET_CHECK(!scene_frames_or_empty.empty())
<< "If |cropped_frames| != nullptr, scene_frames_or_empty must not be "
mediapipe/examples/desktop/autoflip/quality/scene_cropper.h
+2 -2
View File
@@ -60,7 +60,7 @@ class SceneCropper {
// on the transform matrix if |cropped_frames| is not nullptr and
// |scene_frames_or_empty| isn't empty.
// TODO: split this function into two separate functions.
mediapipe::Status CropFrames(
absl::Status CropFrames(
const SceneKeyFrameCropSummary& scene_summary,
const std::vector<int64>& scene_timestamps,
const std::vector<bool>& is_key_frames,
@@ -71,7 +71,7 @@ class SceneCropper {
const bool continue_last_scene, std::vector<cv::Rect>* crop_from_location,
std::vector<cv::Mat>* cropped_frames);
mediapipe::Status ProcessKinematicPathSolver(
absl::Status ProcessKinematicPathSolver(
const SceneKeyFrameCropSummary& scene_summary,
const std::vector<int64>& scene_timestamps,
const std::vector<bool>& is_key_frames,
mediapipe/examples/desktop/autoflip/quality/scene_cropping_viz.cc
+6 -6
View File
@@ -46,7 +46,7 @@ const cv::Scalar kOrange =
cv::Scalar(255.0, 165.0, 0.0); // ica object detector
const cv::Scalar kWhite = cv::Scalar(255.0, 255.0, 255.0); // others
mediapipe::Status DrawDetectionsAndCropRegions(
absl::Status DrawDetectionsAndCropRegions(
const std::vector<cv::Mat>& scene_frames,
const std::vector<bool>& is_key_frames,
const std::vector<KeyFrameInfo>& key_frame_infos,
@@ -130,7 +130,7 @@ mediapipe::Status DrawDetectionsAndCropRegions(
}
viz_frames->push_back(std::move(viz_frame));
}
return mediapipe::OkStatus();
return absl::OkStatus();
}
namespace {
@@ -147,7 +147,7 @@ cv::Rect LimitBounds(const cv::Rect& rect, const int max_width,
}
} // namespace
mediapipe::Status DrawDetectionAndFramingWindow(
absl::Status DrawDetectionAndFramingWindow(
const std::vector<cv::Mat>& org_scene_frames,
const std::vector<cv::Rect>& crop_from_locations,
const ImageFormat::Format image_format, const float overlay_opacity,
@@ -166,10 +166,10 @@ mediapipe::Status DrawDetectionAndFramingWindow(
scene_frame(crop_from_bounded).copyTo(darkened(crop_from_bounded));
viz_frames->push_back(std::move(viz_frame));
}
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status DrawFocusPointAndCropWindow(
absl::Status DrawFocusPointAndCropWindow(
const std::vector<cv::Mat>& scene_frames,
const std::vector<FocusPointFrame>& focus_point_frames,
const float overlay_opacity, const int crop_window_width,
@@ -215,7 +215,7 @@ mediapipe::Status DrawFocusPointAndCropWindow(
}
viz_frames->push_back(std::move(viz_frame));
}
return mediapipe::OkStatus();
return absl::OkStatus();
}
} // namespace autoflip
mediapipe/examples/desktop/autoflip/quality/scene_cropping_viz.h
+3 -3
View File
@@ -36,7 +36,7 @@ namespace autoflip {
// magenta, logos are red, ocrs are yellow (foreground) and light yellow
// (background), brain objects are cyan, ica objects are orange, and the rest
// are white.
mediapipe::Status DrawDetectionsAndCropRegions(
absl::Status DrawDetectionsAndCropRegions(
const std::vector<cv::Mat>& scene_frames,
const std::vector<bool>& is_key_frames,
const std::vector<KeyFrameInfo>& key_frame_infos,
@@ -47,7 +47,7 @@ mediapipe::Status DrawDetectionsAndCropRegions(
// Draws the focus point from the given FocusPointFrame and the crop window
// centered around it on the scene frame in red. This helps visualize the input
// to the retargeter.
mediapipe::Status DrawFocusPointAndCropWindow(
absl::Status DrawFocusPointAndCropWindow(
const std::vector<cv::Mat>& scene_frames,
const std::vector<FocusPointFrame>& focus_point_frames,
const float overlay_opacity, const int crop_window_width,
@@ -57,7 +57,7 @@ mediapipe::Status DrawFocusPointAndCropWindow(
// Draws the final smoothed path of the camera retargeter by darkening the
// removed areas.
mediapipe::Status DrawDetectionAndFramingWindow(
absl::Status DrawDetectionAndFramingWindow(
const std::vector<cv::Mat>& org_scene_frames,
const std::vector<cv::Rect>& crop_from_locations,
const ImageFormat::Format image_format, const float overlay_opacity,
mediapipe/examples/desktop/autoflip/quality/utils.cc
+33 -34
View File
@@ -53,12 +53,12 @@ void NormalizedRectToRect(const RectF& normalized_location, const int width,
ScaleRect(normalized_location, width, height, location);
}
mediapipe::Status ClampRect(const int width, const int height, Rect* location) {
absl::Status ClampRect(const int width, const int height, Rect* location) {
return ClampRect(0, 0, width, height, location);
}
mediapipe::Status ClampRect(const int x0, const int y0, const int x1,
const int y1, Rect* location) {
absl::Status ClampRect(const int x0, const int y0, const int x1, const int y1,
Rect* location) {
RET_CHECK(!(location->x() >= x1 || location->x() + location->width() <= x0 ||
location->y() >= y1 || location->y() + location->height() <= y0));
@@ -73,7 +73,7 @@ mediapipe::Status ClampRect(const int x0, const int y0, const int x1,
location->set_y(clamped_top);
location->set_width(std::max(0, clamped_right - clamped_left));
location->set_height(std::max(0, clamped_bottom - clamped_top));
return mediapipe::OkStatus();
return absl::OkStatus();
}
void RectUnion(const Rect& rect_to_add, Rect* rect) {
@@ -89,13 +89,13 @@ void RectUnion(const Rect& rect_to_add, Rect* rect) {
rect->set_height(y2 - y1);
}
mediapipe::Status PackKeyFrameInfo(const int64 frame_timestamp_ms,
const DetectionSet& detections,
const int original_frame_width,
const int original_frame_height,
const int feature_frame_width,
const int feature_frame_height,
KeyFrameInfo* key_frame_info) {
absl::Status PackKeyFrameInfo(const int64 frame_timestamp_ms,
const DetectionSet& detections,
const int original_frame_width,
const int original_frame_height,
const int feature_frame_width,
const int feature_frame_height,
KeyFrameInfo* key_frame_info) {
RET_CHECK(key_frame_info != nullptr) << "KeyFrameInfo is null";
RET_CHECK(original_frame_width > 0 && original_frame_height > 0 &&
feature_frame_width > 0 && feature_frame_height > 0)
@@ -135,13 +135,12 @@ mediapipe::Status PackKeyFrameInfo(const int64 frame_timestamp_ms,
}
}
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status SortDetections(
const DetectionSet& detections,
std::vector<SalientRegion>* required_regions,
std::vector<SalientRegion>* non_required_regions) {
absl::Status SortDetections(const DetectionSet& detections,
std::vector<SalientRegion>* required_regions,
std::vector<SalientRegion>* non_required_regions) {
required_regions->clear();
non_required_regions->clear();
@@ -174,13 +173,13 @@ mediapipe::Status SortDetections(
non_required_regions->push_back(detections.detections(original_idx));
}
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status SetKeyFrameCropTarget(const int frame_width,
const int frame_height,
const double target_aspect_ratio,
KeyFrameCropOptions* crop_options) {
absl::Status SetKeyFrameCropTarget(const int frame_width,
const int frame_height,
const double target_aspect_ratio,
KeyFrameCropOptions* crop_options) {
RET_CHECK_NE(crop_options, nullptr) << "KeyFrameCropOptions is null.";
RET_CHECK_GT(frame_width, 0) << "Frame width is non-positive.";
RET_CHECK_GT(frame_height, 0) << "Frame height is non-positive.";
@@ -198,10 +197,10 @@ mediapipe::Status SetKeyFrameCropTarget(const int frame_width,
: std::round(frame_width / target_aspect_ratio);
crop_options->set_target_width(crop_target_width);
crop_options->set_target_height(crop_target_height);
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status AggregateKeyFrameResults(
absl::Status AggregateKeyFrameResults(
const KeyFrameCropOptions& key_frame_crop_options,
const std::vector<KeyFrameCropResult>& key_frame_crop_results,
const int scene_frame_width, const int scene_frame_height,
@@ -231,7 +230,7 @@ mediapipe::Status AggregateKeyFrameResults(
// Handles the corner case of no key frames.
if (num_key_frames == 0) {
scene_summary->set_has_salient_region(false);
return mediapipe::OkStatus();
return absl::OkStatus();
}
scene_summary->set_num_key_frames(num_key_frames);
@@ -327,10 +326,10 @@ mediapipe::Status AggregateKeyFrameResults(
scene_summary->key_frame_center_min_y()) /
scene_frame_height;
scene_summary->set_vertical_motion_amount(motion_y);
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status ComputeSceneStaticBordersSize(
absl::Status ComputeSceneStaticBordersSize(
const std::vector<StaticFeatures>& static_features, int* top_border_size,
int* bottom_border_size) {
RET_CHECK(top_border_size) << "Output top border size is null.";
@@ -374,10 +373,10 @@ mediapipe::Status ComputeSceneStaticBordersSize(
*top_border_size = std::max(0, *top_border_size);
*bottom_border_size = std::max(0, *bottom_border_size);
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status FindSolidBackgroundColor(
absl::Status FindSolidBackgroundColor(
const std::vector<StaticFeatures>& static_features,
const std::vector<int64>& static_features_timestamps,
const double min_fraction_solid_background_color,
@@ -422,13 +421,13 @@ mediapipe::Status FindSolidBackgroundColor(
min_fraction_solid_background_color) {
*has_solid_background = true;
}
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status AffineRetarget(const cv::Size& output_size,
const std::vector<cv::Mat>& frames,
const std::vector<cv::Mat>& affine_projection,
std::vector<cv::Mat>* cropped_frames) {
absl::Status AffineRetarget(const cv::Size& output_size,
const std::vector<cv::Mat>& frames,
const std::vector<cv::Mat>& affine_projection,
std::vector<cv::Mat>* cropped_frames) {
RET_CHECK(frames.size() == affine_projection.size())
<< "number of frames and retarget offsets must be the same.";
RET_CHECK(cropped_frames->size() == frames.size())
@@ -442,7 +441,7 @@ mediapipe::Status AffineRetarget(const cv::Size& output_size,
RET_CHECK(affine.rows == 2) << "Affine matrix must be 2x3";
cv::warpAffine(frames[i], (*cropped_frames)[i], affine, output_size);
}
return mediapipe::OkStatus();
return absl::OkStatus();
}
} // namespace autoflip
} // namespace mediapipe
mediapipe/examples/desktop/autoflip/quality/utils.h
+24 -25
View File
@@ -29,31 +29,30 @@ namespace autoflip {
// Packs detected features and timestamp (ms) into a KeyFrameInfo object. Scales
// features back to the original frame size if features have been detected on a
// different frame size.
mediapipe::Status PackKeyFrameInfo(const int64 frame_timestamp_ms,
const DetectionSet& detections,
const int original_frame_width,
const int original_frame_height,
const int feature_frame_width,
const int feature_frame_height,
KeyFrameInfo* key_frame_info);
absl::Status PackKeyFrameInfo(const int64 frame_timestamp_ms,
const DetectionSet& detections,
const int original_frame_width,
const int original_frame_height,
const int feature_frame_width,
const int feature_frame_height,
KeyFrameInfo* key_frame_info);
// Sorts required and non-required salient regions given a detection set.
mediapipe::Status SortDetections(
const DetectionSet& detections,
std::vector<SalientRegion>* required_regions,
std::vector<SalientRegion>* non_required_regions);
absl::Status SortDetections(const DetectionSet& detections,
std::vector<SalientRegion>* required_regions,
std::vector<SalientRegion>* non_required_regions);
// Sets the target crop size in KeyFrameCropOptions based on frame size and
// target aspect ratio so that the target crop size covers the biggest area
// possible in the frame.
mediapipe::Status SetKeyFrameCropTarget(const int frame_width,
const int frame_height,
const double target_aspect_ratio,
KeyFrameCropOptions* crop_options);
absl::Status SetKeyFrameCropTarget(const int frame_width,
const int frame_height,
const double target_aspect_ratio,
KeyFrameCropOptions* crop_options);
// Aggregates information from KeyFrameInfos and KeyFrameCropResults into
// SceneKeyFrameCropSummary.
mediapipe::Status AggregateKeyFrameResults(
absl::Status AggregateKeyFrameResults(
const KeyFrameCropOptions& key_frame_crop_options,
const std::vector<KeyFrameCropResult>& key_frame_crop_results,
const int scene_frame_width, const int scene_frame_height,
@@ -61,7 +60,7 @@ mediapipe::Status AggregateKeyFrameResults(
// Computes the static top and border size across a scene given a vector of
// StaticFeatures over frames.
mediapipe::Status ComputeSceneStaticBordersSize(
absl::Status ComputeSceneStaticBordersSize(
const std::vector<StaticFeatures>& static_features, int* top_border_size,
int* bottom_border_size);
@@ -70,7 +69,7 @@ mediapipe::Status ComputeSceneStaticBordersSize(
// background color exceeds given threshold, i.e.,
// min_fraction_solid_background_color. Builds the background color
// interpolation functions in Lab space using input timestamps.
mediapipe::Status FindSolidBackgroundColor(
absl::Status FindSolidBackgroundColor(
const std::vector<StaticFeatures>& static_features,
const std::vector<int64>& static_features_timestamps,
const double min_fraction_solid_background_color,
@@ -93,12 +92,12 @@ void NormalizedRectToRect(const RectF& normalized_location, const int width,
// Clamps a rectangle to lie within [x0, y0] and [x1, y1]. Returns true if the
// rectangle has any overlapping with the target window.
mediapipe::Status ClampRect(const int x0, const int y0, const int x1,
const int y1, Rect* location);
absl::Status ClampRect(const int x0, const int y0, const int x1, const int y1,
Rect* location);
// Convenience function to clamp a rectangle to lie within [0, 0] and
// [width, height].
mediapipe::Status ClampRect(const int width, const int height, Rect* location);
absl::Status ClampRect(const int width, const int height, Rect* location);
// Enlarges a given rectangle to cover a new rectangle to be added.
void RectUnion(const Rect& rect_to_add, Rect* rect);
@@ -106,10 +105,10 @@ void RectUnion(const Rect& rect_to_add, Rect* rect);
// Performs an affine retarget on a list of input images. Output vector
// cropped_frames must be filled with Mats of the same size as output_size and
// type.
mediapipe::Status AffineRetarget(const cv::Size& output_size,
const std::vector<cv::Mat>& frames,
const std::vector<cv::Mat>& affine_projection,
std::vector<cv::Mat>* cropped_frames);
absl::Status AffineRetarget(const cv::Size& output_size,
const std::vector<cv::Mat>& frames,
const std::vector<cv::Mat>& affine_projection,
std::vector<cv::Mat>* cropped_frames);
} // namespace autoflip
} // namespace mediapipe
mediapipe/examples/desktop/autoflip/quality/visual_scorer.cc
+10 -10
View File
@@ -48,9 +48,9 @@ void CropRectToMat(const cv::Mat& image, cv::Rect* rect) {
VisualScorer::VisualScorer(const VisualScorerOptions& options)
: options_(options) {}
mediapipe::Status VisualScorer::CalculateScore(const cv::Mat& image,
const SalientRegion& region,
float* score) const {
absl::Status VisualScorer::CalculateScore(const cv::Mat& image,
const SalientRegion& region,
float* score) const {
const float weight_sum = options_.area_weight() +
options_.sharpness_weight() +
options_.colorfulness_weight();
@@ -74,7 +74,7 @@ mediapipe::Status VisualScorer::CalculateScore(const cv::Mat& image,
CropRectToMat(image, &region_rect);
if (region_rect.area() == 0) {
*score = 0;
return mediapipe::OkStatus();
return absl::OkStatus();
}
// Compute a score based on area covered by this region.
@@ -108,11 +108,11 @@ mediapipe::Status VisualScorer::CalculateScore(const cv::Mat& image,
if (*score > 1.0f || *score < 0.0f) {
LOG(WARNING) << "Score of region outside expected range: " << *score;
}
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status VisualScorer::CalculateColorfulness(
const cv::Mat& image, float* colorfulness) const {
absl::Status VisualScorer::CalculateColorfulness(const cv::Mat& image,
float* colorfulness) const {
// Convert the image to HSV.
cv::Mat image_hsv;
cv::cvtColor(image, image_hsv, CV_RGB2HSV);
@@ -134,7 +134,7 @@ mediapipe::Status VisualScorer::CalculateColorfulness(
// If the mask is empty, return.
if (empty_mask) {
*colorfulness = 0;
return mediapipe::OkStatus();
return absl::OkStatus();
}
// Generate a 2D histogram (hue/saturation).
@@ -162,7 +162,7 @@ mediapipe::Status VisualScorer::CalculateColorfulness(
}
if (hue_sum == 0.0f) {
*colorfulness = 0;
return mediapipe::OkStatus();
return absl::OkStatus();
}
// Compute the histogram entropy.
@@ -175,7 +175,7 @@ mediapipe::Status VisualScorer::CalculateColorfulness(
}
*colorfulness /= std::log(2.0f);
return mediapipe::OkStatus();
return absl::OkStatus();
}
} // namespace autoflip
mediapipe/examples/desktop/autoflip/quality/visual_scorer.h
+4 -5
View File
@@ -30,13 +30,12 @@ class VisualScorer {
explicit VisualScorer(const VisualScorerOptions& options);
// Computes a score on a salientregion and returns a value [0...1].
mediapipe::Status CalculateScore(const cv::Mat& image,
const SalientRegion& region,
float* score) const;
absl::Status CalculateScore(const cv::Mat& image, const SalientRegion& region,
float* score) const;
private:
mediapipe::Status CalculateColorfulness(const cv::Mat& image,
float* colorfulness) const;
absl::Status CalculateColorfulness(const cv::Mat& image,
float* colorfulness) const;
VisualScorerOptions options_;
};
mediapipe/examples/desktop/demo_run_graph_main.cc
+8 -7
View File
@@ -40,10 +40,11 @@ DEFINE_string(output_video_path, "",
"Full path of where to save result (.mp4 only). "
"If not provided, show result in a window.");
mediapipe::Status RunMPPGraph() {
absl::Status RunMPPGraph() {
std::string calculator_graph_config_contents;
MP_RETURN_IF_ERROR(mediapipe::file::GetContents(
FLAGS_calculator_graph_config_file, &calculator_graph_config_contents));
absl::GetFlag(FLAGS_calculator_graph_config_file),
&calculator_graph_config_contents));
LOG(INFO) << "Get calculator graph config contents: "
<< calculator_graph_config_contents;
mediapipe::CalculatorGraphConfig config =
@@ -56,16 +57,16 @@ mediapipe::Status RunMPPGraph() {
LOG(INFO) << "Initialize the camera or load the video.";
cv::VideoCapture capture;
const bool load_video = !FLAGS_input_video_path.empty();
const bool load_video = !absl::GetFlag(FLAGS_input_video_path).empty();
if (load_video) {
capture.open(FLAGS_input_video_path);
capture.open(absl::GetFlag(FLAGS_input_video_path));
} else {
capture.open(0);
}
RET_CHECK(capture.isOpened());
cv::VideoWriter writer;
const bool save_video = !FLAGS_output_video_path.empty();
const bool save_video = !absl::GetFlag(FLAGS_output_video_path).empty();
if (!save_video) {
cv::namedWindow(kWindowName, /*flags=WINDOW_AUTOSIZE*/ 1);
#if (CV_MAJOR_VERSION >= 3) && (CV_MINOR_VERSION >= 2)
@@ -125,7 +126,7 @@ mediapipe::Status RunMPPGraph() {
if (save_video) {
if (!writer.isOpened()) {
LOG(INFO) << "Prepare video writer.";
writer.open(FLAGS_output_video_path,
writer.open(absl::GetFlag(FLAGS_output_video_path),
mediapipe::fourcc('a', 'v', 'c', '1'), // .mp4
capture.get(cv::CAP_PROP_FPS), output_frame_mat.size());
RET_CHECK(writer.isOpened());
@@ -148,7 +149,7 @@ mediapipe::Status RunMPPGraph() {
int main(int argc, char** argv) {
google::InitGoogleLogging(argv[0]);
gflags::ParseCommandLineFlags(&argc, &argv, true);
mediapipe::Status run_status = RunMPPGraph();
absl::Status run_status = RunMPPGraph();
if (!run_status.ok()) {
LOG(ERROR) << "Failed to run the graph: " << run_status.message();
return EXIT_FAILURE;
mediapipe/examples/desktop/demo_run_graph_main_gpu.cc
+14 -13
View File
@@ -44,10 +44,11 @@ DEFINE_string(output_video_path, "",
"Full path of where to save result (.mp4 only). "
"If not provided, show result in a window.");
mediapipe::Status RunMPPGraph() {
absl::Status RunMPPGraph() {
std::string calculator_graph_config_contents;
MP_RETURN_IF_ERROR(mediapipe::file::GetContents(
FLAGS_calculator_graph_config_file, &calculator_graph_config_contents));
absl::GetFlag(FLAGS_calculator_graph_config_file),
&calculator_graph_config_contents));
LOG(INFO) << "Get calculator graph config contents: "
<< calculator_graph_config_contents;
mediapipe::CalculatorGraphConfig config =
@@ -66,16 +67,16 @@ mediapipe::Status RunMPPGraph() {
LOG(INFO) << "Initialize the camera or load the video.";
cv::VideoCapture capture;
const bool load_video = !FLAGS_input_video_path.empty();
const bool load_video = !absl::GetFlag(FLAGS_input_video_path).empty();
if (load_video) {
capture.open(FLAGS_input_video_path);
capture.open(absl::GetFlag(FLAGS_input_video_path));
} else {
capture.open(0);
}
RET_CHECK(capture.isOpened());
cv::VideoWriter writer;
const bool save_video = !FLAGS_output_video_path.empty();
const bool save_video = !absl::GetFlag(FLAGS_output_video_path).empty();
if (!save_video) {
cv::namedWindow(kWindowName, /*flags=WINDOW_AUTOSIZE*/ 1);
#if (CV_MAJOR_VERSION >= 3) && (CV_MINOR_VERSION >= 2)
@@ -122,7 +123,7 @@ mediapipe::Status RunMPPGraph() {
(double)cv::getTickCount() / (double)cv::getTickFrequency() * 1e6;
MP_RETURN_IF_ERROR(
gpu_helper.RunInGlContext([&input_frame, &frame_timestamp_us, &graph,
&gpu_helper]() -> ::mediapipe::Status {
&gpu_helper]() -> absl::Status {
// Convert ImageFrame to GpuBuffer.
auto texture = gpu_helper.CreateSourceTexture(*input_frame.get());
auto gpu_frame = texture.GetFrame<mediapipe::GpuBuffer>();
@@ -132,7 +133,7 @@ mediapipe::Status RunMPPGraph() {
MP_RETURN_IF_ERROR(graph.AddPacketToInputStream(
kInputStream, mediapipe::Adopt(gpu_frame.release())
.At(mediapipe::Timestamp(frame_timestamp_us))));
return mediapipe::OkStatus();
return absl::OkStatus();
}));
// Get the graph result packet, or stop if that fails.
@@ -142,7 +143,7 @@ mediapipe::Status RunMPPGraph() {
// Convert GpuBuffer to ImageFrame.
MP_RETURN_IF_ERROR(gpu_helper.RunInGlContext(
[&packet, &output_frame, &gpu_helper]() -> ::mediapipe::Status {
[&packet, &output_frame, &gpu_helper]() -> absl::Status {
auto& gpu_frame = packet.Get<mediapipe::GpuBuffer>();
auto texture = gpu_helper.CreateSourceTexture(gpu_frame);
output_frame = absl::make_unique<mediapipe::ImageFrame>(
@@ -150,13 +151,13 @@ mediapipe::Status RunMPPGraph() {
gpu_frame.width(), gpu_frame.height(),
mediapipe::ImageFrame::kGlDefaultAlignmentBoundary);
gpu_helper.BindFramebuffer(texture);
const auto info =
mediapipe::GlTextureInfoForGpuBufferFormat(gpu_frame.format(), 0);
const auto info = mediapipe::GlTextureInfoForGpuBufferFormat(
gpu_frame.format(), 0, gpu_helper.GetGlVersion());
glReadPixels(0, 0, texture.width(), texture.height(), info.gl_format,
info.gl_type, output_frame->MutablePixelData());
glFlush();
texture.Release();
return mediapipe::OkStatus();
return absl::OkStatus();
}));
// Convert back to opencv for display or saving.
@@ -168,7 +169,7 @@ mediapipe::Status RunMPPGraph() {
if (save_video) {
if (!writer.isOpened()) {
LOG(INFO) << "Prepare video writer.";
writer.open(FLAGS_output_video_path,
writer.open(absl::GetFlag(FLAGS_output_video_path),
mediapipe::fourcc('a', 'v', 'c', '1'), // .mp4
capture.get(cv::CAP_PROP_FPS), output_frame_mat.size());
RET_CHECK(writer.isOpened());
@@ -191,7 +192,7 @@ mediapipe::Status RunMPPGraph() {
int main(int argc, char** argv) {
google::InitGoogleLogging(argv[0]);
gflags::ParseCommandLineFlags(&argc, &argv, true);
mediapipe::Status run_status = RunMPPGraph();
absl::Status run_status = RunMPPGraph();
if (!run_status.ok()) {
LOG(ERROR) << "Failed to run the graph: " << run_status.message();
return EXIT_FAILURE;
mediapipe/examples/desktop/hello_world/hello_world.cc
+1 -1
View File
@@ -21,7 +21,7 @@
namespace mediapipe {
mediapipe::Status PrintHelloWorld() {
absl::Status PrintHelloWorld() {
// Configures a simple graph, which concatenates 2 PassThroughCalculators.
CalculatorGraphConfig config = ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
input_stream: "in"
mediapipe/examples/desktop/iris_tracking/iris_depth_from_image_desktop.cc
+12 -14
View File
@@ -47,18 +47,16 @@ DEFINE_string(output_image_path, "",
namespace {
mediapipe::StatusOr<std::string> ReadFileToString(
const std::string& file_path) {
absl::StatusOr<std::string> ReadFileToString(const std::string& file_path) {
std::string contents;
MP_RETURN_IF_ERROR(mediapipe::file::GetContents(file_path, &contents));
return contents;
}
mediapipe::Status ProcessImage(
std::unique_ptr<mediapipe::CalculatorGraph> graph) {
absl::Status ProcessImage(std::unique_ptr<mediapipe::CalculatorGraph> graph) {
LOG(INFO) << "Load the image.";
ASSIGN_OR_RETURN(const std::string raw_image,
ReadFileToString(FLAGS_input_image_path));
ReadFileToString(absl::GetFlag(FLAGS_input_image_path)));
LOG(INFO) << "Start running the calculator graph.";
ASSIGN_OR_RETURN(mediapipe::OutputStreamPoller output_image_poller,
@@ -80,7 +78,7 @@ mediapipe::Status ProcessImage(
// Get the graph result packets, or stop if that fails.
mediapipe::Packet left_iris_depth_packet;
if (!left_iris_depth_poller.Next(&left_iris_depth_packet)) {
return mediapipe::UnknownError(
return absl::UnknownError(
"Failed to get packet from output stream 'left_iris_depth_mm'.");
}
const auto& left_iris_depth_mm = left_iris_depth_packet.Get<float>();
@@ -89,7 +87,7 @@ mediapipe::Status ProcessImage(
mediapipe::Packet right_iris_depth_packet;
if (!right_iris_depth_poller.Next(&right_iris_depth_packet)) {
return mediapipe::UnknownError(
return absl::UnknownError(
"Failed to get packet from output stream 'right_iris_depth_mm'.");
}
const auto& right_iris_depth_mm = right_iris_depth_packet.Get<float>();
@@ -99,7 +97,7 @@ mediapipe::Status ProcessImage(
mediapipe::Packet output_image_packet;
if (!output_image_poller.Next(&output_image_packet)) {
return mediapipe::UnknownError(
return absl::UnknownError(
"Failed to get packet from output stream 'output_image'.");
}
auto& output_frame = output_image_packet.Get<mediapipe::ImageFrame>();
@@ -107,10 +105,10 @@ mediapipe::Status ProcessImage(
// Convert back to opencv for display or saving.
cv::Mat output_frame_mat = mediapipe::formats::MatView(&output_frame);
cv::cvtColor(output_frame_mat, output_frame_mat, cv::COLOR_RGB2BGR);
const bool save_image = !FLAGS_output_image_path.empty();
const bool save_image = !absl::GetFlag(FLAGS_output_image_path).empty();
if (save_image) {
LOG(INFO) << "Saving image to file...";
cv::imwrite(FLAGS_output_image_path, output_frame_mat);
cv::imwrite(absl::GetFlag(FLAGS_output_image_path), output_frame_mat);
} else {
cv::namedWindow(kWindowName, /*flags=WINDOW_AUTOSIZE*/ 1);
cv::imshow(kWindowName, output_frame_mat);
@@ -123,7 +121,7 @@ mediapipe::Status ProcessImage(
return graph->WaitUntilDone();
}
mediapipe::Status RunMPPGraph() {
absl::Status RunMPPGraph() {
std::string calculator_graph_config_contents;
MP_RETURN_IF_ERROR(mediapipe::file::GetContents(
kCalculatorGraphConfigFile, &calculator_graph_config_contents));
@@ -138,11 +136,11 @@ mediapipe::Status RunMPPGraph() {
absl::make_unique<mediapipe::CalculatorGraph>();
MP_RETURN_IF_ERROR(graph->Initialize(config));
const bool load_image = !FLAGS_input_image_path.empty();
const bool load_image = !absl::GetFlag(FLAGS_input_image_path).empty();
if (load_image) {
return ProcessImage(std::move(graph));
} else {
return mediapipe::InvalidArgumentError("Missing image file.");
return absl::InvalidArgumentError("Missing image file.");
}
}
@@ -151,7 +149,7 @@ mediapipe::Status RunMPPGraph() {
int main(int argc, char** argv) {
google::InitGoogleLogging(argv[0]);
gflags::ParseCommandLineFlags(&argc, &argv, true);
mediapipe::Status run_status = RunMPPGraph();
absl::Status run_status = RunMPPGraph();
if (!run_status.ok()) {
LOG(ERROR) << "Failed to run the graph: " << run_status.message();
return EXIT_FAILURE;
mediapipe/examples/desktop/media_sequence/run_graph_file_io_main.cc
+9 -8
View File
@@ -38,10 +38,11 @@ DEFINE_string(output_side_packets, "",
"side packets and paths to write to disk for the "
"CalculatorGraph.");
mediapipe::Status RunMPPGraph() {
absl::Status RunMPPGraph() {
std::string calculator_graph_config_contents;
MP_RETURN_IF_ERROR(mediapipe::file::GetContents(
FLAGS_calculator_graph_config_file, &calculator_graph_config_contents));
absl::GetFlag(FLAGS_calculator_graph_config_file),
&calculator_graph_config_contents));
LOG(INFO) << "Get calculator graph config contents: "
<< calculator_graph_config_contents;
mediapipe::CalculatorGraphConfig config =
@@ -49,7 +50,7 @@ mediapipe::Status RunMPPGraph() {
calculator_graph_config_contents);
std::map<std::string, mediapipe::Packet> input_side_packets;
std::vector<std::string> kv_pairs =
absl::StrSplit(FLAGS_input_side_packets, ',');
absl::StrSplit(absl::GetFlag(FLAGS_input_side_packets), ',');
for (const std::string& kv_pair : kv_pairs) {
std::vector<std::string> name_and_value = absl::StrSplit(kv_pair, '=');
RET_CHECK(name_and_value.size() == 2);
@@ -66,26 +67,26 @@ mediapipe::Status RunMPPGraph() {
LOG(INFO) << "Start running the calculator graph.";
MP_RETURN_IF_ERROR(graph.Run());
LOG(INFO) << "Gathering output side packets.";
kv_pairs = absl::StrSplit(FLAGS_output_side_packets, ',');
kv_pairs = absl::StrSplit(absl::GetFlag(FLAGS_output_side_packets), ',');
for (const std::string& kv_pair : kv_pairs) {
std::vector<std::string> name_and_value = absl::StrSplit(kv_pair, '=');
RET_CHECK(name_and_value.size() == 2);
mediapipe::StatusOr<mediapipe::Packet> output_packet =
absl::StatusOr<mediapipe::Packet> output_packet =
graph.GetOutputSidePacket(name_and_value[0]);
RET_CHECK(output_packet.ok())
<< "Packet " << name_and_value[0] << " was not available.";
const std::string& serialized_string =
output_packet.ValueOrDie().Get<std::string>();
output_packet.value().Get<std::string>();
MP_RETURN_IF_ERROR(
mediapipe::file::SetContents(name_and_value[1], serialized_string));
}
return mediapipe::OkStatus();
return absl::OkStatus();
}
int main(int argc, char** argv) {
google::InitGoogleLogging(argv[0]);
gflags::ParseCommandLineFlags(&argc, &argv, true);
mediapipe::Status run_status = RunMPPGraph();
absl::Status run_status = RunMPPGraph();
if (!run_status.ok()) {
LOG(ERROR) << "Failed to run the graph: " << run_status.message();
return EXIT_FAILURE;
mediapipe/examples/desktop/object_detection_3d/BUILD
+3 -3
View File
@@ -22,9 +22,9 @@ package(default_visibility = ["//mediapipe/examples:__subpackages__"])
# --calculator_graph_config_file=mediapipe/graphs/object_detection_3d/objectron_desktop_cpu.pbtxt \
# --input_side_packets="input_video_path=<input_video_path>,box_landmark_model_path=mediapipe/models/object_detection_3d_sneakers.tflite,output_video_path=<output_video_path>,allowed_labels=Footwear"
# To detect objects from other categories, change box_landmark_model_path and allowed_labels accordingly.
# Chair: box_landmark_model_path=mediapipe/models/object_detection_3d_chair.tflite,allowed_labels=Chair
# Camera: box_landmark_model_path=mediapipe/models/object_detection_3d_camera.tflite,allowed_labels=Camera
# Cup: box_landmark_model_path=mediapipe/models/object_detection_3d_cup.tflite,allowed_labels=Mug
# Chair: box_landmark_model_path=mediapipe/modules/objectron/object_detection_3d_chair.tflite,allowed_labels=Chair
# Camera: box_landmark_model_path=mediapipe/modules/objectron/object_detection_3d_camera.tflite,allowed_labels=Camera
# Cup: box_landmark_model_path=mediapipe/modules/objectron/object_detection_3d_cup.tflite,allowed_labels=Mug
cc_binary(
name = "objectron_cpu",
deps = [
mediapipe/examples/desktop/simple_run_graph_main.cc
+24 -23
View File
@@ -58,31 +58,29 @@ DEFINE_string(output_side_packets_file, "",
"The name of the local file to output all side packets specified "
"with --output_side_packets. ");
mediapipe::Status OutputStreamToLocalFile(
mediapipe::OutputStreamPoller& poller) {
absl::Status OutputStreamToLocalFile(mediapipe::OutputStreamPoller& poller) {
std::ofstream file;
file.open(FLAGS_output_stream_file);
file.open(absl::GetFlag(FLAGS_output_stream_file));
mediapipe::Packet packet;
while (poller.Next(&packet)) {
std::string output_data;
if (!FLAGS_strip_timestamps) {
if (!absl::GetFlag(FLAGS_strip_timestamps)) {
absl::StrAppend(&output_data, packet.Timestamp().Value(), ",");
}
absl::StrAppend(&output_data, packet.Get<std::string>(), "\n");
file << output_data;
}
file.close();
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status OutputSidePacketsToLocalFile(
mediapipe::CalculatorGraph& graph) {
if (!FLAGS_output_side_packets.empty() &&
!FLAGS_output_side_packets_file.empty()) {
absl::Status OutputSidePacketsToLocalFile(mediapipe::CalculatorGraph& graph) {
if (!absl::GetFlag(FLAGS_output_side_packets).empty() &&
!absl::GetFlag(FLAGS_output_side_packets_file).empty()) {
std::ofstream file;
file.open(FLAGS_output_side_packets_file);
file.open(absl::GetFlag(FLAGS_output_side_packets_file));
std::vector<std::string> side_packet_names =
absl::StrSplit(FLAGS_output_side_packets, ',');
absl::StrSplit(absl::GetFlag(FLAGS_output_side_packets), ',');
for (const std::string& side_packet_name : side_packet_names) {
ASSIGN_OR_RETURN(auto status_or_packet,
graph.GetOutputSidePacket(side_packet_name));
@@ -91,27 +89,28 @@ mediapipe::Status OutputSidePacketsToLocalFile(
}
file.close();
} else {
RET_CHECK(FLAGS_output_side_packets.empty() &&
FLAGS_output_side_packets_file.empty())
RET_CHECK(absl::GetFlag(FLAGS_output_side_packets).empty() &&
absl::GetFlag(FLAGS_output_side_packets_file).empty())
<< "--output_side_packets and --output_side_packets_file should be "
"specified in pair.";
}
return mediapipe::OkStatus();
return absl::OkStatus();
}
mediapipe::Status RunMPPGraph() {
absl::Status RunMPPGraph() {
std::string calculator_graph_config_contents;
MP_RETURN_IF_ERROR(mediapipe::file::GetContents(
FLAGS_calculator_graph_config_file, &calculator_graph_config_contents));
absl::GetFlag(FLAGS_calculator_graph_config_file),
&calculator_graph_config_contents));
LOG(INFO) << "Get calculator graph config contents: "
<< calculator_graph_config_contents;
mediapipe::CalculatorGraphConfig config =
mediapipe::ParseTextProtoOrDie<mediapipe::CalculatorGraphConfig>(
calculator_graph_config_contents);
std::map<std::string, mediapipe::Packet> input_side_packets;
if (!FLAGS_input_side_packets.empty()) {
if (!absl::GetFlag(FLAGS_input_side_packets).empty()) {
std::vector<std::string> kv_pairs =
absl::StrSplit(FLAGS_input_side_packets, ',');
absl::StrSplit(absl::GetFlag(FLAGS_input_side_packets), ',');
for (const std::string& kv_pair : kv_pairs) {
std::vector<std::string> name_and_value = absl::StrSplit(kv_pair, '=');
RET_CHECK(name_and_value.size() == 2);
@@ -123,14 +122,16 @@ mediapipe::Status RunMPPGraph() {
LOG(INFO) << "Initialize the calculator graph.";
mediapipe::CalculatorGraph graph;
MP_RETURN_IF_ERROR(graph.Initialize(config, input_side_packets));
if (!FLAGS_output_stream.empty() && !FLAGS_output_stream_file.empty()) {
ASSIGN_OR_RETURN(auto poller,
graph.AddOutputStreamPoller(FLAGS_output_stream));
if (!absl::GetFlag(FLAGS_output_stream).empty() &&
!absl::GetFlag(FLAGS_output_stream_file).empty()) {
ASSIGN_OR_RETURN(auto poller, graph.AddOutputStreamPoller(
absl::GetFlag(FLAGS_output_stream)));
LOG(INFO) << "Start running the calculator graph.";
MP_RETURN_IF_ERROR(graph.StartRun({}));
MP_RETURN_IF_ERROR(OutputStreamToLocalFile(poller));
} else {
RET_CHECK(FLAGS_output_stream.empty() && FLAGS_output_stream_file.empty())
RET_CHECK(absl::GetFlag(FLAGS_output_stream).empty() &&
absl::GetFlag(FLAGS_output_stream_file).empty())
<< "--output_stream and --output_stream_file should be specified in "
"pair.";
LOG(INFO) << "Start running the calculator graph.";
@@ -143,7 +144,7 @@ mediapipe::Status RunMPPGraph() {
int main(int argc, char** argv) {
google::InitGoogleLogging(argv[0]);
gflags::ParseCommandLineFlags(&argc, &argv, true);
mediapipe::Status run_status = RunMPPGraph();
absl::Status run_status = RunMPPGraph();
if (!run_status.ok()) {
LOG(ERROR) << "Failed to run the graph: " << run_status.message();
return EXIT_FAILURE;
mediapipe/examples/desktop/youtube8m/BUILD
-1
View File
@@ -21,7 +21,6 @@ cc_binary(
"@com_google_absl//absl/strings",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/formats:matrix",
"//mediapipe/framework/formats:matrix_data_cc_proto",
"//mediapipe/framework/port:commandlineflags",
"//mediapipe/framework/port:file_helpers",
"//mediapipe/framework/port:map_util",
mediapipe/examples/desktop/youtube8m/extract_yt8m_features.cc
+9 -8
View File
@@ -39,10 +39,11 @@ DEFINE_string(output_side_packets, "",
"side packets and paths to write to disk for the "
"CalculatorGraph.");
mediapipe::Status RunMPPGraph() {
absl::Status RunMPPGraph() {
std::string calculator_graph_config_contents;
MP_RETURN_IF_ERROR(mediapipe::file::GetContents(
FLAGS_calculator_graph_config_file, &calculator_graph_config_contents));
absl::GetFlag(FLAGS_calculator_graph_config_file),
&calculator_graph_config_contents));
LOG(INFO) << "Get calculator graph config contents: "
<< calculator_graph_config_contents;
mediapipe::CalculatorGraphConfig config =
@@ -50,7 +51,7 @@ mediapipe::Status RunMPPGraph() {
calculator_graph_config_contents);
std::map<std::string, mediapipe::Packet> input_side_packets;
std::vector<std::string> kv_pairs =
absl::StrSplit(FLAGS_input_side_packets, ',');
absl::StrSplit(absl::GetFlag(FLAGS_input_side_packets), ',');
for (const std::string& kv_pair : kv_pairs) {
std::vector<std::string> name_and_value = absl::StrSplit(kv_pair, '=');
RET_CHECK(name_and_value.size() == 2);
@@ -107,26 +108,26 @@ mediapipe::Status RunMPPGraph() {
LOG(INFO) << "Start running the calculator graph.";
MP_RETURN_IF_ERROR(graph.Run());
LOG(INFO) << "Gathering output side packets.";
kv_pairs = absl::StrSplit(FLAGS_output_side_packets, ',');
kv_pairs = absl::StrSplit(absl::GetFlag(FLAGS_output_side_packets), ',');
for (const std::string& kv_pair : kv_pairs) {
std::vector<std::string> name_and_value = absl::StrSplit(kv_pair, '=');
RET_CHECK(name_and_value.size() == 2);
mediapipe::StatusOr<mediapipe::Packet> output_packet =
absl::StatusOr<mediapipe::Packet> output_packet =
graph.GetOutputSidePacket(name_and_value[0]);
RET_CHECK(output_packet.ok())
<< "Packet " << name_and_value[0] << " was not available.";
const std::string& serialized_string =
output_packet.ValueOrDie().Get<std::string>();
output_packet.value().Get<std::string>();
MP_RETURN_IF_ERROR(
mediapipe::file::SetContents(name_and_value[1], serialized_string));
}
return mediapipe::OkStatus();
return absl::OkStatus();
}
int main(int argc, char** argv) {
google::InitGoogleLogging(argv[0]);
gflags::ParseCommandLineFlags(&argc, &argv, true);
mediapipe::Status run_status = RunMPPGraph();
absl::Status run_status = RunMPPGraph();
if (!run_status.ok()) {
LOG(ERROR) << "Failed to run the graph: " << run_status.message();
return EXIT_FAILURE;
mediapipe/examples/ios/common/CommonViewController.mm
+4 -1
View File
@@ -137,10 +137,10 @@ static const char* kVideoQueueLabel = "com.google.mediapipe.example.videoQueue";
[self.cameraSource requestCameraAccessWithCompletionHandler:^void(BOOL granted) {
if (granted) {
[self startGraphAndCamera];
dispatch_async(dispatch_get_main_queue(), ^{
self.noCameraLabel.hidden = YES;
});
[self startGraphAndCamera];
}
}];
@@ -155,6 +155,9 @@ static const char* kVideoQueueLabel = "com.google.mediapipe.example.videoQueue";
if (![self.mediapipeGraph startWithError:&error]) {
NSLog(@"Failed to start graph: %@", error);
}
else if (![self.mediapipeGraph waitUntilIdleWithError:&error]) {
NSLog(@"Failed to complete graph initial run: %@", error);
}
// Start fetching frames from the camera.
dispatch_async(self.videoQueue, ^{
mediapipe/examples/ios/faceeffect/BUILD
+15
View File
@@ -57,6 +57,21 @@ objc_library(
"FaceEffectViewController.h",
],
copts = ["-std=c++17"],
data = [
"Base.lproj/LaunchScreen.storyboard",
"Base.lproj/Main.storyboard",
"//mediapipe/graphs/face_effect:face_effect_gpu.binarypb",
"//mediapipe/graphs/face_effect/data:axis.binarypb",
"//mediapipe/graphs/face_effect/data:axis.pngblob",
"//mediapipe/graphs/face_effect/data:facepaint.pngblob",
"//mediapipe/graphs/face_effect/data:glasses.binarypb",
"//mediapipe/graphs/face_effect/data:glasses.pngblob",
"//mediapipe/modules/face_detection:face_detection_front.tflite",
"//mediapipe/modules/face_geometry/data:geometry_pipeline_metadata.binarypb",
"//mediapipe/modules/face_geometry/data:geometry_pipeline_metadata_detection.binarypb",
"//mediapipe/modules/face_geometry/data:geometry_pipeline_metadata_landmarks.binarypb",
"//mediapipe/modules/face_landmark:face_landmark.tflite",
],
sdk_frameworks = [
"AVFoundation",
"CoreGraphics",
mediapipe/examples/ios/faceeffect/FaceEffectViewController.mm
+52 -12
View File
@@ -18,6 +18,8 @@
#import "mediapipe/objc/MPPGraph.h"
#import "mediapipe/objc/MPPLayerRenderer.h"
#include <map>
#include <string>
#include <utility>
#include "mediapipe/framework/formats/matrix_data.pb.h"
@@ -27,13 +29,19 @@
static NSString* const kGraphName = @"face_effect_gpu";
static const char* kInputStream = "input_video";
static const char* kIsFacepaintEffectSelectedInputStream = "is_facepaint_effect_selected";
static const char* kOutputStream = "output_video";
static const char* kMultiFaceGeometryStream = "multi_face_geometry";
static const char* kVideoQueueLabel = "com.google.mediapipe.example.videoQueue";
static const char* kSelectedEffectIdInputStream = "selected_effect_id";
static const char* kUseFaceDetectionInputSourceInputSidePacket = "use_face_detection_input_source";
static const BOOL kUseFaceDetectionInputSource = NO;
static const int kMatrixTranslationZIndex = 14;
static const int kSelectedEffectIdAxis = 0;
static const int kSelectedEffectIdFacepaint = 1;
static const int kSelectedEffectIdGlasses = 2;
@interface FaceEffectViewController () <MPPGraphDelegate, MPPInputSourceDelegate>
// The MediaPipe graph currently in use. Initialized in viewDidLoad, started in viewWillAppear: and
@@ -45,7 +53,7 @@ static const int kMatrixTranslationZIndex = 14;
@implementation FaceEffectViewController {
/// Handle tap gestures.
UITapGestureRecognizer* _tapGestureRecognizer;
BOOL _isFacepaintEffectSelected;
int _selectedEffectId;
/// Handles camera access via AVCaptureSession library.
MPPCameraInputSource* _cameraSource;
@@ -93,8 +101,14 @@ static const int kMatrixTranslationZIndex = 14;
mediapipe::CalculatorGraphConfig config;
config.ParseFromArray(data.bytes, data.length);
// Pass the kUseFaceDetectionInputSource flag value as an input side packet into the graph.
std::map<std::string, mediapipe::Packet> side_packets;
side_packets[kUseFaceDetectionInputSourceInputSidePacket] =
mediapipe::MakePacket<bool>(kUseFaceDetectionInputSource);
// Create MediaPipe graph with mediapipe::CalculatorGraphConfig proto object.
MPPGraph* newGraph = [[MPPGraph alloc] initWithGraphConfig:config];
[newGraph addSidePackets:side_packets];
[newGraph addFrameOutputStream:kOutputStream outputPacketType:MPPPacketTypePixelBuffer];
[newGraph addFrameOutputStream:kMultiFaceGeometryStream outputPacketType:MPPPacketTypeRaw];
return newGraph;
@@ -110,8 +124,13 @@ static const int kMatrixTranslationZIndex = 14;
action:@selector(handleTap)];
[self.view addGestureRecognizer:_tapGestureRecognizer];
// By default, render the glasses effect.
_isFacepaintEffectSelected = NO;
// By default, render the axis effect for the face detection input source and the glasses effect
// for the face landmark input source.
if (kUseFaceDetectionInputSource) {
_selectedEffectId = kSelectedEffectIdAxis;
} else {
_selectedEffectId = kSelectedEffectIdGlasses;
}
_renderer = [[MPPLayerRenderer alloc] init];
_renderer.layer.frame = _liveView.layer.bounds;
@@ -175,7 +194,28 @@ static const int kMatrixTranslationZIndex = 14;
// multiple pre-bundled face effects without a need to recompile the app.
- (void)handleTap {
dispatch_async(_videoQueue, ^{
_isFacepaintEffectSelected = !_isFacepaintEffectSelected;
// Avoid switching the Axis effect for the face detection input source.
if (kUseFaceDetectionInputSource) {
return;
}
// Looped effect order: glasses -> facepaint -> axis -> glasses -> ...
switch (_selectedEffectId) {
case kSelectedEffectIdAxis: {
_selectedEffectId = kSelectedEffectIdGlasses;
break;
}
case kSelectedEffectIdFacepaint: {
_selectedEffectId = kSelectedEffectIdAxis;
break;
}
case kSelectedEffectIdGlasses: {
_selectedEffectId = kSelectedEffectIdFacepaint;
break;
}
}
});
}
@@ -189,7 +229,7 @@ static const int kMatrixTranslationZIndex = 14;
// Display the captured image on the screen.
CVPixelBufferRetain(pixelBuffer);
dispatch_async(dispatch_get_main_queue(), ^{
_effectSwitchingHintLabel.hidden = NO;
_effectSwitchingHintLabel.hidden = kUseFaceDetectionInputSource;
[_renderer renderPixelBuffer:pixelBuffer];
CVPixelBufferRelease(pixelBuffer);
});
@@ -236,18 +276,18 @@ static const int kMatrixTranslationZIndex = 14;
mediapipe::Timestamp graphTimestamp(static_cast<mediapipe::TimestampBaseType>(
mediapipe::Timestamp::kTimestampUnitsPerSecond * CMTimeGetSeconds(timestamp)));
mediapipe::Packet isFacepaintEffectSelectedPacket =
mediapipe::MakePacket<bool>(_isFacepaintEffectSelected).At(graphTimestamp);
mediapipe::Packet selectedEffectIdPacket =
mediapipe::MakePacket<int>(_selectedEffectId).At(graphTimestamp);
[self.graph sendPixelBuffer:imageBuffer
intoStream:kInputStream
packetType:MPPPacketTypePixelBuffer
timestamp:graphTimestamp];
// Alongside the input camera frame, we also send the `is_facepaint_effect_selected` boolean
// packet to indicate which effect should be rendered on this frame.
[self.graph movePacket:std::move(isFacepaintEffectSelectedPacket)
intoStream:kIsFacepaintEffectSelectedInputStream
// Alongside the input camera frame, we also send the `selected_effect_id` int packet to indicate
// which effect should be rendered on this frame.
[self.graph movePacket:std::move(selectedEffectIdPacket)
intoStream:kSelectedEffectIdInputStream
error:nil];
}
mediapipe/examples/ios/objectdetectiontrackinggpu/BUILD
+70
View File
@@ -0,0 +1,70 @@
# Copyright 2019 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
load(
"@build_bazel_rules_apple//apple:ios.bzl",
"ios_application",
)
load(
"//mediapipe/examples/ios:bundle_id.bzl",
"BUNDLE_ID_PREFIX",
"example_provisioning",
)
licenses(["notice"])
MIN_IOS_VERSION = "10.0"
alias(
name = "objectdetectiontrackinggpu",
actual = "ObjectDetectionTrackingGpuApp",
)
ios_application(
name = "ObjectDetectionTrackingGpuApp",
app_icons = ["//mediapipe/examples/ios/common:AppIcon"],
bundle_id = BUNDLE_ID_PREFIX + ".ObjectDetectionTrackingGpu",
families = [
"iphone",
"ipad",
],
infoplists = [
"//mediapipe/examples/ios/common:Info.plist",
"Info.plist",
],
minimum_os_version = MIN_IOS_VERSION,
provisioning_profile = example_provisioning(),
deps = [
":ObjectDetectionTrackingGpuAppLibrary",
"@ios_opencv//:OpencvFramework",
],
)
objc_library(
name = "ObjectDetectionTrackingGpuAppLibrary",
data = [
"//mediapipe/graphs/tracking:mobile_gpu_binary_graph",
"//mediapipe/models:ssdlite_object_detection.tflite",
"//mediapipe/models:ssdlite_object_detection_labelmap.txt",
],
deps = [
"//mediapipe/examples/ios/common:CommonMediaPipeAppLibrary",
] + select({
"//mediapipe:ios_i386": [],
"//mediapipe:ios_x86_64": [],
"//conditions:default": [
"//mediapipe/graphs/tracking:mobile_calculators",
],
}),
)
mediapipe/examples/ios/objectdetectiontrackinggpu/Info.plist
+14
View File
@@ -0,0 +1,14 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>CameraPosition</key>
<string>back</string>
<key>GraphName</key>
<string>mobile_gpu</string>
<key>GraphOutputStream</key>
<string>output_video</string>
<key>GraphInputStream</key>
<string>input_video</string>
</dict>
</plist>