Project import generated by Copybara.
GitOrigin-RevId: e3a43e4e5e519cd14df7095749059e2613bdcf76
This commit is contained in:
MediaPipe Team
@@ -1,4 +1 @@
|
||||
MediaPipe Examples
|
||||
==================
|
||||
|
||||
This directory contains MediaPipe Android example applications. Please see [src/java/com/google/mediapipe/apps/README.md](src/java/com/google/mediapipe/apps/README.md) for details.
|
||||
This directory contains MediaPipe example applications for Android. Please see [Solutions](https://solutions.mediapipe.dev)for details.
|
||||
|
||||
@@ -1,7 +0,0 @@
|
||||
tricorder: {
|
||||
options: {
|
||||
builder: {
|
||||
config: "android_arm64"
|
||||
}
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -83,7 +83,7 @@ android_binary(
|
||||
manifest = "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:AndroidManifest.xml",
|
||||
manifest_values = {
|
||||
"applicationId": "com.google.mediapipe.apps.objectdetection3d",
|
||||
"appName": "Object Detection 3D",
|
||||
"appName": "Objectron",
|
||||
"mainActivity": ".MainActivity",
|
||||
"cameraFacingFront": "False",
|
||||
"binaryGraphName": "object_detection_3d.binarypb",
|
||||
|
||||
@@ -1,113 +1 @@
|
||||
**Hello World**
|
||||
|
||||
To build the "Hello World" example, use:
|
||||
|
||||
```
|
||||
bazel build -c opt mediapipe/examples/desktop/hello_world:hello_world
|
||||
```
|
||||
|
||||
and then run it using:
|
||||
|
||||
```
|
||||
export GLOG_logtostderr=1
|
||||
|
||||
bazel-bin/mediapipe/examples/desktop/hello_world/hello_world
|
||||
```
|
||||
|
||||
**TFlite Object Detection**
|
||||
|
||||
To build the object detection demo using a TFLite model on desktop, use:
|
||||
|
||||
```
|
||||
bazel build -c opt mediapipe/examples/desktop/object_detection:object_detection_tflite --define MEDIAPIPE_DISABLE_GPU=1
|
||||
```
|
||||
|
||||
and run it using:
|
||||
|
||||
```
|
||||
export GLOG_logtostderr=1
|
||||
|
||||
bazel-bin/mediapipe/examples/desktop/object_detection/object_detection_tflite \
|
||||
--calculator_graph_config_file=mediapipe/graphs/object_detection/object_detection_desktop_tflite_graph.pbtxt \
|
||||
--input_side_packets=input_video_path=/path/to/input/file,output_video_path=/path/to/output/file
|
||||
```
|
||||
|
||||
**TensorFlow Object Detection**
|
||||
|
||||
To build the object detection demo using a TensorFlow model on desktop, use:
|
||||
|
||||
```
|
||||
export GLOG_logtostderr=1
|
||||
|
||||
bazel build -c opt mediapipe/examples/desktop/object_detection:object_detection_tensorflow \
|
||||
--define MEDIAPIPE_DISABLE_GPU=1
|
||||
```
|
||||
|
||||
and run it using:
|
||||
|
||||
```
|
||||
export GLOG_logtostderr=1
|
||||
|
||||
bazel-bin/mediapipe/examples/desktop/object_detection/object_detection_tensorflow \
|
||||
--calculator_graph_config_file=mediapipe/graphs/object_detection/object_detection_desktop_tensorflow_graph.pbtxt \
|
||||
--input_side_packets=input_video_path=/path/to/input/file,output_video_path=/path/to/output/file
|
||||
```
|
||||
|
||||
**TFlite Hand Detection**
|
||||
|
||||
To build the hand detection demo using a TFLite model on desktop, use:
|
||||
|
||||
```
|
||||
bazel build -c opt mediapipe/examples/desktop/hand_tracking:hand_tracking_tflite --define MEDIAPIPE_DISABLE_GPU=1
|
||||
```
|
||||
|
||||
and run it using:
|
||||
|
||||
```
|
||||
export GLOG_logtostderr=1
|
||||
|
||||
bazel-bin/mediapipe/examples/desktop/hand_tracking/hand_tracking_tflite \
|
||||
--calculator_graph_config_file=mediapipe/graphs/hand_tracking/hand_detection_desktop.pbtxt \
|
||||
--input_side_packets=input_video_path=/path/to/input/file,output_video_path=/path/to/output/file
|
||||
```
|
||||
|
||||
**TFlite Hand Tracking**
|
||||
|
||||
To build the hand tracking demo using a TFLite model on desktop, use:
|
||||
|
||||
```
|
||||
bazel build -c opt mediapipe/examples/desktop/hand_tracking:hand_tracking_tflite --define MEDIAPIPE_DISABLE_GPU=1
|
||||
```
|
||||
|
||||
and run it using:
|
||||
|
||||
```
|
||||
export GLOG_logtostderr=1
|
||||
|
||||
bazel-bin/mediapipe/examples/desktop/hand_tracking/hand_tracking_tflite \
|
||||
--calculator_graph_config_file=mediapipe/graphs/hand_tracking/hand_tracking_desktop.pbtxt \
|
||||
--input_side_packets=input_video_path=/path/to/input/file,output_video_path=/path/to/output/file
|
||||
```
|
||||
|
||||
**TFlite Multi-Hand Tracking**
|
||||
|
||||
To build the multi-hand tracking demo using a TFLite model on desktop, use:
|
||||
|
||||
```
|
||||
bazel build -c opt mediapipe/examples/desktop/multi_hand_tracking:multi_hand_tracking_tflite --define MEDIAPIPE_DISABLE_GPU=1
|
||||
```
|
||||
|
||||
and run it using:
|
||||
|
||||
```
|
||||
export GLOG_logtostderr=1
|
||||
|
||||
bazel-bin/mediapipe/examples/desktop/multi_hand_tracking/multi_hand_tracking_tflite \
|
||||
--calculator_graph_config_file=mediapipe/graphs/hand_tracking/multi_hand_tracking_desktop.pbtxt \
|
||||
--input_side_packets=input_video_path=/path/to/input/file,output_video_path=/path/to/output/file
|
||||
```
|
||||
|
||||
To change the number of hands to `x` in this application, change:
|
||||
|
||||
1. `min_size:x` in `CollectionHasMinSizeCalculatorOptions` in `mediapipe/graphs/hand_tracking/multi_hand_tracking_desktop.pbtxt`.
|
||||
2. `max_vec_size:x` in `ClipVectorSizeCalculatorOptions` in `mediapipe/examples/dekstop/hand_tracking/subgraphs/multi_hand_detection_cpu.pbtxt`.
|
||||
This directory contains MediaPipe example applications for desktop. Please see [Solutions](https://solutions.mediapipe.dev)for details.
|
||||
|
||||
@@ -62,8 +62,10 @@ cc_library(
|
||||
"//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto",
|
||||
"//mediapipe/examples/desktop/autoflip/quality:kinematic_path_solver",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/formats:detection_cc_proto",
|
||||
"//mediapipe/framework/formats:image_frame",
|
||||
"//mediapipe/framework/formats:image_frame_opencv",
|
||||
"//mediapipe/framework/formats:location_data_cc_proto",
|
||||
"//mediapipe/framework/formats:rect_cc_proto",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/port:status",
|
||||
],
|
||||
@@ -126,17 +128,20 @@ cc_test(
|
||||
":content_zooming_calculator",
|
||||
":content_zooming_calculator_cc_proto",
|
||||
"//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto",
|
||||
"//mediapipe/examples/desktop/autoflip/quality:kinematic_path_solver",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:calculator_runner",
|
||||
"//mediapipe/framework/formats:detection_cc_proto",
|
||||
"//mediapipe/framework/formats:image_frame",
|
||||
"//mediapipe/framework/formats:image_frame_opencv",
|
||||
"//mediapipe/framework/formats:location_data_cc_proto",
|
||||
"//mediapipe/framework/formats:rect_cc_proto",
|
||||
"//mediapipe/framework/port:benchmark",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"//mediapipe/framework/port:opencv_core",
|
||||
"//mediapipe/framework/port:parse_text_proto",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/port:status",
|
||||
"@com_google_absl//absl/strings",
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
@@ -19,16 +19,20 @@
|
||||
#include "mediapipe/examples/desktop/autoflip/calculators/content_zooming_calculator.pb.h"
|
||||
#include "mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/formats/detection.pb.h"
|
||||
#include "mediapipe/framework/formats/image_frame.h"
|
||||
#include "mediapipe/framework/formats/image_frame_opencv.h"
|
||||
#include "mediapipe/framework/formats/location_data.pb.h"
|
||||
#include "mediapipe/framework/formats/rect.pb.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
#include "mediapipe/framework/port/status_builder.h"
|
||||
|
||||
constexpr char kVideoFrame[] = "VIDEO";
|
||||
constexpr char kVideoSize[] = "VIDEO_SIZE";
|
||||
constexpr char kDetectionSet[] = "DETECTIONS";
|
||||
constexpr char kSalientRegions[] = "SALIENT_REGIONS";
|
||||
constexpr char kDetections[] = "DETECTIONS";
|
||||
constexpr char kDetectedBorders[] = "BORDERS";
|
||||
constexpr char kCropRect[] = "CROP_RECT";
|
||||
// Field-of-view (degrees) of the camera's x-axis (width).
|
||||
// TODO: Parameterize FOV based on camera specs.
|
||||
constexpr float kWidthFieldOfView = 60;
|
||||
@@ -37,12 +41,12 @@ namespace mediapipe {
|
||||
namespace autoflip {
|
||||
|
||||
// Content zooming calculator zooms in on content when a detection has
|
||||
// "only_required" set true. It does this by computing the value of top/bottom
|
||||
// borders to remove from the output and sends these to the
|
||||
// SceneCroppingCalculator. When more than one detections are received the zoom
|
||||
// box is calculated as the union of the detections. Typical applications
|
||||
// include mobile makeover and autofliplive face reframing. Currently only
|
||||
// supports y-dimension zooming.
|
||||
// "only_required" set true or any raw detection input. It does this by
|
||||
// computing the value of top/bottom borders to remove from the output and sends
|
||||
// these to the SceneCroppingCalculator using BORDERS output or a full rect crop
|
||||
// using CROP_RECT output. When more than one detections are received the
|
||||
// zoom box is calculated as the union of the detections. Typical applications
|
||||
// include mobile makeover and autofliplive face reframing.
|
||||
class ContentZoomingCalculator : public CalculatorBase {
|
||||
public:
|
||||
ContentZoomingCalculator()
|
||||
@@ -56,26 +60,32 @@ class ContentZoomingCalculator : public CalculatorBase {
|
||||
::mediapipe::Status Process(mediapipe::CalculatorContext* cc) override;
|
||||
|
||||
private:
|
||||
// Converts bounds to tilt offset and height.
|
||||
::mediapipe::Status ConvertToTiltZoom(float xmin, float xmax, float ymin,
|
||||
float ymax, int* tilt_offset,
|
||||
int* height);
|
||||
// Converts bounds to tilt offset, pan offset and height.
|
||||
::mediapipe::Status ConvertToPanTiltZoom(float xmin, float xmax, float ymin,
|
||||
float ymax, int* tilt_offset,
|
||||
int* pan_offset, int* height);
|
||||
ContentZoomingCalculatorOptions options_;
|
||||
// Detection frame width/height.
|
||||
int frame_height_;
|
||||
int frame_width_;
|
||||
// Path solver used to smooth top/bottom border crop values.
|
||||
std::unique_ptr<KinematicPathSolver> path_solver_height_;
|
||||
std::unique_ptr<KinematicPathSolver> path_solver_width_;
|
||||
std::unique_ptr<KinematicPathSolver> path_solver_offset_;
|
||||
// Are parameters initialized.
|
||||
bool initialized_;
|
||||
// Stores the time of the last "only_required" input.
|
||||
int64 last_only_required_detection_;
|
||||
// Border values of last message with detection.
|
||||
// Rect values of last message with detection(s).
|
||||
int last_measured_height_;
|
||||
int last_measured_x_offset_;
|
||||
int last_measured_y_offset_;
|
||||
// Min border values.
|
||||
float min_height_value_;
|
||||
// Target aspect ratio.
|
||||
float target_aspect_;
|
||||
// Max size of bounding box. If input/output aspect ratios are the same,
|
||||
// will be 1.0. Else, will be less than 1.0 to prevent exceeding the size of
|
||||
// the image in either dimension.
|
||||
float max_frame_value_;
|
||||
};
|
||||
REGISTER_CALCULATOR(ContentZoomingCalculator);
|
||||
|
||||
@@ -92,8 +102,18 @@ REGISTER_CALCULATOR(ContentZoomingCalculator);
|
||||
return ::mediapipe::UnknownErrorBuilder(MEDIAPIPE_LOC)
|
||||
<< "Input VIDEO or VIDEO_SIZE must be provided.";
|
||||
}
|
||||
cc->Inputs().Tag(kDetectionSet).Set<DetectionSet>();
|
||||
cc->Outputs().Tag(kDetectedBorders).Set<StaticFeatures>();
|
||||
if (cc->Inputs().HasTag(kSalientRegions)) {
|
||||
cc->Inputs().Tag(kSalientRegions).Set<DetectionSet>();
|
||||
}
|
||||
if (cc->Inputs().HasTag(kDetections)) {
|
||||
cc->Inputs().Tag(kDetections).Set<std::vector<mediapipe::Detection>>();
|
||||
}
|
||||
if (cc->Outputs().HasTag(kDetectedBorders)) {
|
||||
cc->Outputs().Tag(kDetectedBorders).Set<StaticFeatures>();
|
||||
}
|
||||
if (cc->Outputs().HasTag(kCropRect)) {
|
||||
cc->Outputs().Tag(kCropRect).Set<mediapipe::Rect>();
|
||||
}
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
@@ -108,29 +128,38 @@ REGISTER_CALCULATOR(ContentZoomingCalculator);
|
||||
if (options_.has_min_motion_to_reframe()) {
|
||||
return ::mediapipe::UnknownErrorBuilder(MEDIAPIPE_LOC)
|
||||
<< "Deprecated min_motion_to_reframe was set, please set "
|
||||
"in kinematic_options_zoom and kinematic_options_tilt directly.";
|
||||
"in kinematic_options_zoom and kinematic_options_tilt "
|
||||
"directly.";
|
||||
}
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status ContentZoomingCalculator::ConvertToTiltZoom(
|
||||
::mediapipe::Status ContentZoomingCalculator::ConvertToPanTiltZoom(
|
||||
float xmin, float xmax, float ymin, float ymax, int* tilt_offset,
|
||||
int* height) {
|
||||
int* pan_offset, int* height) {
|
||||
// Find center of the y-axis offset (for tilt control).
|
||||
float y_center = ymin + (ymax - ymin) / 2;
|
||||
// Find center of the x-axis offset (for pan control).
|
||||
float x_center = xmin + (xmax - xmin) / 2;
|
||||
// Find size and apply scale factor to y-axis.
|
||||
float fit_size = fmax((ymax - ymin) / options_.scale_factor(), xmax - xmin);
|
||||
// Apply min zoom for cases where the target size is wider than input frame
|
||||
// size.
|
||||
fit_size = fmin(min_height_value_, fit_size);
|
||||
// Apply max frame for cases where the target size is different than input
|
||||
// frame size.
|
||||
fit_size = fmin(max_frame_value_, fit_size);
|
||||
// Prevent box from extending beyond the image.
|
||||
if (y_center - fit_size / 2 < 0) {
|
||||
y_center = fit_size / 2;
|
||||
} else if (y_center + fit_size / 2 > 1) {
|
||||
y_center = 1 - fit_size / 2;
|
||||
}
|
||||
if (x_center - fit_size / 2 < 0) {
|
||||
x_center = fit_size / 2;
|
||||
} else if (x_center + fit_size / 2 > 1) {
|
||||
x_center = 1 - fit_size / 2;
|
||||
}
|
||||
// Scale to pixel coordinates.
|
||||
*tilt_offset = frame_height_ * y_center;
|
||||
*pan_offset = frame_width_ * x_center;
|
||||
*height = frame_height_ * fit_size;
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
@@ -151,6 +180,20 @@ namespace {
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
::mediapipe::Status UpdateRanges(const mediapipe::Detection& detection,
|
||||
float* xmin, float* xmax, float* ymin,
|
||||
float* ymax) {
|
||||
RET_CHECK(detection.location_data().format() ==
|
||||
mediapipe::LocationData::RELATIVE_BOUNDING_BOX)
|
||||
<< "Face detection input is lacking required relative_bounding_box()";
|
||||
const auto& location = detection.location_data().relative_bounding_box();
|
||||
*xmin = fmin(*xmin, location.xmin());
|
||||
*xmax = fmax(*xmax, location.xmin() + location.width());
|
||||
*ymin = fmin(*ymin, location.ymin());
|
||||
*ymax = fmax(*ymax, location.ymin() + location.height());
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
void MakeStaticFeatures(const int top_border, const int bottom_border,
|
||||
const int frame_width, const int frame_height,
|
||||
StaticFeatures* static_feature) {
|
||||
@@ -173,10 +216,8 @@ void MakeStaticFeatures(const int top_border, const int bottom_border,
|
||||
::mediapipe::Status ContentZoomingCalculator::Process(
|
||||
mediapipe::CalculatorContext* cc) {
|
||||
if (cc->Inputs().HasTag(kVideoFrame)) {
|
||||
cv::Mat frame = mediapipe::formats::MatView(
|
||||
&cc->Inputs().Tag(kVideoFrame).Get<ImageFrame>());
|
||||
frame_width_ = frame.cols;
|
||||
frame_height_ = frame.rows;
|
||||
frame_width_ = cc->Inputs().Tag(kVideoFrame).Get<ImageFrame>().Width();
|
||||
frame_height_ = cc->Inputs().Tag(kVideoFrame).Get<ImageFrame>().Height();
|
||||
} else if (cc->Inputs().HasTag(kVideoSize)) {
|
||||
frame_width_ =
|
||||
cc->Inputs().Tag(kVideoSize).Get<std::pair<int, int>>().first;
|
||||
@@ -191,10 +232,14 @@ void MakeStaticFeatures(const int top_border, const int bottom_border,
|
||||
path_solver_height_ = std::make_unique<KinematicPathSolver>(
|
||||
options_.kinematic_options_zoom(), 0, frame_height_,
|
||||
static_cast<float>(frame_width_) / kWidthFieldOfView);
|
||||
path_solver_width_ = std::make_unique<KinematicPathSolver>(
|
||||
options_.kinematic_options_pan(), 0, frame_width_,
|
||||
static_cast<float>(frame_width_) / kWidthFieldOfView);
|
||||
path_solver_offset_ = std::make_unique<KinematicPathSolver>(
|
||||
options_.kinematic_options_tilt(), 0, frame_height_,
|
||||
static_cast<float>(frame_width_) / kWidthFieldOfView);
|
||||
min_height_value_ = 1.0;
|
||||
max_frame_value_ = 1.0;
|
||||
target_aspect_ = frame_width_ / static_cast<float>(frame_height_);
|
||||
// If target size is set and wider than input aspect, make sure to always
|
||||
// crop the min required amount.
|
||||
if (options_.has_target_size()) {
|
||||
@@ -203,75 +248,107 @@ void MakeStaticFeatures(const int top_border, const int bottom_border,
|
||||
RET_CHECK_GT(options_.target_size().height(), 0)
|
||||
<< "Provided target height not valid.";
|
||||
float input_aspect = frame_width_ / static_cast<float>(frame_height_);
|
||||
float target_aspect = options_.target_size().width() /
|
||||
static_cast<float>(options_.target_size().height());
|
||||
min_height_value_ =
|
||||
(input_aspect < target_aspect) ? input_aspect / target_aspect : 1.0;
|
||||
target_aspect_ = options_.target_size().width() /
|
||||
static_cast<float>(options_.target_size().height());
|
||||
max_frame_value_ = std::min(input_aspect / target_aspect_,
|
||||
target_aspect_ / input_aspect);
|
||||
}
|
||||
last_measured_height_ = min_height_value_ * frame_height_;
|
||||
last_measured_height_ = max_frame_value_ * frame_height_;
|
||||
last_measured_x_offset_ = target_aspect_ * frame_width_;
|
||||
last_measured_y_offset_ = frame_width_ / 2;
|
||||
initialized_ = true;
|
||||
}
|
||||
|
||||
auto detection_set = cc->Inputs().Tag(kDetectionSet).Get<DetectionSet>();
|
||||
bool only_required_found = false;
|
||||
|
||||
// Compute the box that contains all "is_required" detections.
|
||||
float xmin = 1, ymin = 1, xmax = 0, ymax = 0;
|
||||
for (const auto& region : detection_set.detections()) {
|
||||
if (!region.only_required()) {
|
||||
continue;
|
||||
if (cc->Inputs().HasTag(kSalientRegions)) {
|
||||
auto detection_set = cc->Inputs().Tag(kSalientRegions).Get<DetectionSet>();
|
||||
for (const auto& region : detection_set.detections()) {
|
||||
if (!region.only_required()) {
|
||||
continue;
|
||||
}
|
||||
only_required_found = true;
|
||||
MP_RETURN_IF_ERROR(UpdateRanges(region, &xmin, &xmax, &ymin, &ymax));
|
||||
}
|
||||
}
|
||||
|
||||
if (cc->Inputs().HasTag(kDetections)) {
|
||||
auto raw_detections =
|
||||
cc->Inputs().Tag(kDetections).Get<std::vector<mediapipe::Detection>>();
|
||||
for (const auto& detection : raw_detections) {
|
||||
only_required_found = true;
|
||||
MP_RETURN_IF_ERROR(UpdateRanges(detection, &xmin, &xmax, &ymin, &ymax));
|
||||
}
|
||||
only_required_found = true;
|
||||
MP_RETURN_IF_ERROR(UpdateRanges(region, &xmin, &xmax, &ymin, &ymax));
|
||||
}
|
||||
|
||||
// Convert bounds to tilt/zoom and in pixel coordinates.
|
||||
int offset, height;
|
||||
MP_RETURN_IF_ERROR(
|
||||
ConvertToTiltZoom(xmin, xmax, ymin, ymax, &offset, &height));
|
||||
int offset_y, height, offset_x;
|
||||
MP_RETURN_IF_ERROR(ConvertToPanTiltZoom(xmin, xmax, ymin, ymax, &offset_y,
|
||||
&offset_x, &height));
|
||||
|
||||
if (only_required_found) {
|
||||
// A only required detection was found.
|
||||
last_only_required_detection_ = cc->InputTimestamp().Microseconds();
|
||||
last_measured_height_ = height;
|
||||
last_measured_y_offset_ = offset;
|
||||
last_measured_x_offset_ = offset_x;
|
||||
last_measured_y_offset_ = offset_y;
|
||||
} else if (cc->InputTimestamp().Microseconds() -
|
||||
last_only_required_detection_ >=
|
||||
options_.us_before_zoomout()) {
|
||||
// No only_require detections found within salient regions packets arriving
|
||||
// since us_before_zoomout duration.
|
||||
height = min_height_value_ * frame_height_;
|
||||
offset = frame_height_ / 2;
|
||||
// No only_require detections found within salient regions packets
|
||||
// arriving since us_before_zoomout duration.
|
||||
height = max_frame_value_ * frame_height_;
|
||||
offset_x = (target_aspect_ * height) / 2;
|
||||
offset_y = frame_height_ / 2;
|
||||
} else {
|
||||
// No only detection found but using last detection due to
|
||||
// duration_before_zoomout_us setting.
|
||||
height = last_measured_height_;
|
||||
offset = last_measured_y_offset_;
|
||||
offset_x = last_measured_x_offset_;
|
||||
offset_y = last_measured_y_offset_;
|
||||
}
|
||||
|
||||
// Compute smoothed camera paths.
|
||||
MP_RETURN_IF_ERROR(path_solver_height_->AddObservation(
|
||||
height, cc->InputTimestamp().Microseconds()));
|
||||
MP_RETURN_IF_ERROR(path_solver_width_->AddObservation(
|
||||
offset_x, cc->InputTimestamp().Microseconds()));
|
||||
MP_RETURN_IF_ERROR(path_solver_offset_->AddObservation(
|
||||
offset, cc->InputTimestamp().Microseconds()));
|
||||
int path_size;
|
||||
MP_RETURN_IF_ERROR(path_solver_height_->GetState(&path_size));
|
||||
int path_offset;
|
||||
MP_RETURN_IF_ERROR(path_solver_offset_->GetState(&path_offset));
|
||||
offset_y, cc->InputTimestamp().Microseconds()));
|
||||
int path_height;
|
||||
MP_RETURN_IF_ERROR(path_solver_height_->GetState(&path_height));
|
||||
int path_offset_x;
|
||||
MP_RETURN_IF_ERROR(path_solver_width_->GetState(&path_offset_x));
|
||||
int path_offset_y;
|
||||
MP_RETURN_IF_ERROR(path_solver_offset_->GetState(&path_offset_y));
|
||||
|
||||
// Convert to top/bottom borders to remove.
|
||||
int path_top = path_offset - path_size / 2;
|
||||
int path_bottom = frame_height_ - (path_offset + path_size / 2);
|
||||
int path_top = path_offset_y - path_height / 2;
|
||||
int path_bottom = frame_height_ - (path_offset_y + path_height / 2);
|
||||
|
||||
// Transmit result downstream.
|
||||
std::unique_ptr<StaticFeatures> features =
|
||||
absl::make_unique<StaticFeatures>();
|
||||
MakeStaticFeatures(path_top, path_bottom, frame_width_, frame_height_,
|
||||
features.get());
|
||||
cc->Outputs()
|
||||
.Tag(kDetectedBorders)
|
||||
.AddPacket(Adopt(features.release()).At(cc->InputTimestamp()));
|
||||
// Transmit result downstream to scenecroppingcalculator.
|
||||
if (cc->Outputs().HasTag(kDetectedBorders)) {
|
||||
std::unique_ptr<StaticFeatures> features =
|
||||
absl::make_unique<StaticFeatures>();
|
||||
MakeStaticFeatures(path_top, path_bottom, frame_width_, frame_height_,
|
||||
features.get());
|
||||
cc->Outputs()
|
||||
.Tag(kDetectedBorders)
|
||||
.AddPacket(Adopt(features.release()).At(cc->InputTimestamp()));
|
||||
}
|
||||
|
||||
// Transmit downstream to glcroppingcalculator.
|
||||
if (cc->Outputs().HasTag(kCropRect)) {
|
||||
auto gpu_rect = absl::make_unique<mediapipe::Rect>();
|
||||
gpu_rect->set_x_center(path_offset_x);
|
||||
gpu_rect->set_width(path_height * target_aspect_);
|
||||
gpu_rect->set_y_center(path_offset_y);
|
||||
gpu_rect->set_height(path_height);
|
||||
cc->Outputs().Tag(kCropRect).Add(gpu_rect.release(),
|
||||
Timestamp(cc->InputTimestamp()));
|
||||
}
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
@@ -32,6 +32,8 @@ message ContentZoomingCalculatorOptions {
|
||||
optional KinematicOptions kinematic_options_zoom = 6;
|
||||
// Kinematic options for tilt (y-axis reframing.)
|
||||
optional KinematicOptions kinematic_options_tilt = 7;
|
||||
// Kinematic options for pan (x-axis reframing.)
|
||||
optional KinematicOptions kinematic_options_pan = 10;
|
||||
// Duration (in MicroSeconds) before returning to fully zoomed out position
|
||||
// when no "only_required" frames are received.
|
||||
optional int64 us_before_zoomout = 9 [default = 1000000];
|
||||
|
||||
+116
-8
@@ -16,10 +16,14 @@
|
||||
|
||||
#include "mediapipe/examples/desktop/autoflip/autoflip_messages.pb.h"
|
||||
#include "mediapipe/examples/desktop/autoflip/calculators/content_zooming_calculator.pb.h"
|
||||
#include "mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/calculator_runner.h"
|
||||
#include "mediapipe/framework/formats/detection.pb.h"
|
||||
#include "mediapipe/framework/formats/image_frame.h"
|
||||
#include "mediapipe/framework/formats/image_frame_opencv.h"
|
||||
#include "mediapipe/framework/formats/location_data.pb.h"
|
||||
#include "mediapipe/framework/formats/rect.pb.h"
|
||||
#include "mediapipe/framework/port/benchmark.h"
|
||||
#include "mediapipe/framework/port/gmock.h"
|
||||
#include "mediapipe/framework/port/gtest.h"
|
||||
@@ -36,14 +40,14 @@ namespace {
|
||||
const char kConfigA[] = R"(
|
||||
calculator: "ContentZoomingCalculator"
|
||||
input_stream: "VIDEO:camera_frames"
|
||||
input_stream: "DETECTIONS:detection_set"
|
||||
input_stream: "SALIENT_REGIONS:detection_set"
|
||||
output_stream: "BORDERS:borders"
|
||||
)";
|
||||
|
||||
const char kConfigB[] = R"(
|
||||
calculator: "ContentZoomingCalculator"
|
||||
input_stream: "VIDEO:camera_frames"
|
||||
input_stream: "DETECTIONS:detection_set"
|
||||
input_stream: "SALIENT_REGIONS:detection_set"
|
||||
output_stream: "BORDERS:borders"
|
||||
options: {
|
||||
[mediapipe.autoflip.ContentZoomingCalculatorOptions.ext]: {
|
||||
@@ -58,10 +62,17 @@ const char kConfigB[] = R"(
|
||||
const char kConfigC[] = R"(
|
||||
calculator: "ContentZoomingCalculator"
|
||||
input_stream: "VIDEO_SIZE:size"
|
||||
input_stream: "DETECTIONS:detection_set"
|
||||
input_stream: "SALIENT_REGIONS:detection_set"
|
||||
output_stream: "BORDERS:borders"
|
||||
)";
|
||||
|
||||
const char kConfigD[] = R"(
|
||||
calculator: "ContentZoomingCalculator"
|
||||
input_stream: "VIDEO_SIZE:size"
|
||||
input_stream: "DETECTIONS:detections"
|
||||
output_stream: "CROP_RECT:rect"
|
||||
)";
|
||||
|
||||
void CheckBorder(const StaticFeatures& static_features, int width, int height,
|
||||
int top_border, int bottom_border) {
|
||||
ASSERT_EQ(2, static_features.border().size());
|
||||
@@ -80,6 +91,43 @@ void CheckBorder(const StaticFeatures& static_features, int width, int height,
|
||||
EXPECT_EQ(Border::BOTTOM, part.relative_position());
|
||||
}
|
||||
|
||||
void AddDetection(const cv::Rect_<float>& position, const int64 time,
|
||||
CalculatorRunner* runner) {
|
||||
auto detections = std::make_unique<std::vector<mediapipe::Detection>>();
|
||||
mediapipe::Detection detection;
|
||||
detection.mutable_location_data()->set_format(
|
||||
mediapipe::LocationData::RELATIVE_BOUNDING_BOX);
|
||||
detection.mutable_location_data()
|
||||
->mutable_relative_bounding_box()
|
||||
->set_height(position.height);
|
||||
detection.mutable_location_data()->mutable_relative_bounding_box()->set_width(
|
||||
position.width);
|
||||
detection.mutable_location_data()->mutable_relative_bounding_box()->set_xmin(
|
||||
position.x);
|
||||
detection.mutable_location_data()->mutable_relative_bounding_box()->set_ymin(
|
||||
position.y);
|
||||
detections->push_back(detection);
|
||||
runner->MutableInputs()
|
||||
->Tag("DETECTIONS")
|
||||
.packets.push_back(Adopt(detections.release()).At(Timestamp(time)));
|
||||
|
||||
auto input_size = ::absl::make_unique<std::pair<int, int>>(1000, 1000);
|
||||
runner->MutableInputs()
|
||||
->Tag("VIDEO_SIZE")
|
||||
.packets.push_back(Adopt(input_size.release()).At(Timestamp(time)));
|
||||
}
|
||||
|
||||
void CheckCropRect(const int x_center, const int y_center, const int width,
|
||||
const int height, const int frame_number,
|
||||
const std::vector<Packet>& output_packets) {
|
||||
ASSERT_GT(output_packets.size(), frame_number);
|
||||
const auto& rect = output_packets[frame_number].Get<mediapipe::Rect>();
|
||||
EXPECT_EQ(rect.x_center(), x_center);
|
||||
EXPECT_EQ(rect.y_center(), y_center);
|
||||
EXPECT_EQ(rect.width(), width);
|
||||
EXPECT_EQ(rect.height(), height);
|
||||
}
|
||||
|
||||
TEST(ContentZoomingCalculatorTest, ZoomTest) {
|
||||
auto runner = ::absl::make_unique<CalculatorRunner>(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfigA));
|
||||
@@ -98,7 +146,7 @@ TEST(ContentZoomingCalculatorTest, ZoomTest) {
|
||||
Adopt(input_frame.release()).At(Timestamp(0)));
|
||||
|
||||
runner->MutableInputs()
|
||||
->Tag("DETECTIONS")
|
||||
->Tag("SALIENT_REGIONS")
|
||||
.packets.push_back(Adopt(detection_set.release()).At(Timestamp(0)));
|
||||
|
||||
// Run the calculator.
|
||||
@@ -111,6 +159,66 @@ TEST(ContentZoomingCalculatorTest, ZoomTest) {
|
||||
CheckBorder(static_features, 1000, 1000, 495, 395);
|
||||
}
|
||||
|
||||
TEST(ContentZoomingCalculatorTest, ZoomTestFullPTZ) {
|
||||
auto runner = ::absl::make_unique<CalculatorRunner>(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfigD));
|
||||
AddDetection(cv::Rect_<float>(.4, .5, .1, .1), 0, runner.get());
|
||||
MP_ASSERT_OK(runner->Run());
|
||||
CheckCropRect(450, 550, 111, 111, 0,
|
||||
runner->Outputs().Tag("CROP_RECT").packets);
|
||||
}
|
||||
|
||||
TEST(ContentZoomingCalculatorTest, PanConfig) {
|
||||
auto config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfigD);
|
||||
auto* options = config.mutable_options()->MutableExtension(
|
||||
ContentZoomingCalculatorOptions::ext);
|
||||
options->mutable_kinematic_options_pan()->set_min_motion_to_reframe(0.0);
|
||||
options->mutable_kinematic_options_tilt()->set_min_motion_to_reframe(5.0);
|
||||
options->mutable_kinematic_options_zoom()->set_min_motion_to_reframe(5.0);
|
||||
auto runner = ::absl::make_unique<CalculatorRunner>(config);
|
||||
AddDetection(cv::Rect_<float>(.4, .5, .1, .1), 0, runner.get());
|
||||
AddDetection(cv::Rect_<float>(.45, .55, .15, .15), 1000000, runner.get());
|
||||
MP_ASSERT_OK(runner->Run());
|
||||
CheckCropRect(450, 550, 111, 111, 0,
|
||||
runner->Outputs().Tag("CROP_RECT").packets);
|
||||
CheckCropRect(488, 550, 111, 111, 1,
|
||||
runner->Outputs().Tag("CROP_RECT").packets);
|
||||
}
|
||||
|
||||
TEST(ContentZoomingCalculatorTest, TiltConfig) {
|
||||
auto config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfigD);
|
||||
auto* options = config.mutable_options()->MutableExtension(
|
||||
ContentZoomingCalculatorOptions::ext);
|
||||
options->mutable_kinematic_options_pan()->set_min_motion_to_reframe(5.0);
|
||||
options->mutable_kinematic_options_tilt()->set_min_motion_to_reframe(0.0);
|
||||
options->mutable_kinematic_options_zoom()->set_min_motion_to_reframe(5.0);
|
||||
auto runner = ::absl::make_unique<CalculatorRunner>(config);
|
||||
AddDetection(cv::Rect_<float>(.4, .5, .1, .1), 0, runner.get());
|
||||
AddDetection(cv::Rect_<float>(.45, .55, .15, .15), 1000000, runner.get());
|
||||
MP_ASSERT_OK(runner->Run());
|
||||
CheckCropRect(450, 550, 111, 111, 0,
|
||||
runner->Outputs().Tag("CROP_RECT").packets);
|
||||
CheckCropRect(450, 588, 111, 111, 1,
|
||||
runner->Outputs().Tag("CROP_RECT").packets);
|
||||
}
|
||||
|
||||
TEST(ContentZoomingCalculatorTest, ZoomConfig) {
|
||||
auto config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfigD);
|
||||
auto* options = config.mutable_options()->MutableExtension(
|
||||
ContentZoomingCalculatorOptions::ext);
|
||||
options->mutable_kinematic_options_pan()->set_min_motion_to_reframe(5.0);
|
||||
options->mutable_kinematic_options_tilt()->set_min_motion_to_reframe(5.0);
|
||||
options->mutable_kinematic_options_zoom()->set_min_motion_to_reframe(0.0);
|
||||
auto runner = ::absl::make_unique<CalculatorRunner>(config);
|
||||
AddDetection(cv::Rect_<float>(.4, .5, .1, .1), 0, runner.get());
|
||||
AddDetection(cv::Rect_<float>(.45, .55, .15, .15), 1000000, runner.get());
|
||||
MP_ASSERT_OK(runner->Run());
|
||||
CheckCropRect(450, 550, 111, 111, 0,
|
||||
runner->Outputs().Tag("CROP_RECT").packets);
|
||||
CheckCropRect(450, 550, 139, 139, 1,
|
||||
runner->Outputs().Tag("CROP_RECT").packets);
|
||||
}
|
||||
|
||||
TEST(ContentZoomingCalculatorTest, MinAspectBorderValues) {
|
||||
auto runner = ::absl::make_unique<CalculatorRunner>(
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfigB));
|
||||
@@ -129,7 +237,7 @@ TEST(ContentZoomingCalculatorTest, MinAspectBorderValues) {
|
||||
Adopt(input_frame.release()).At(Timestamp(0)));
|
||||
|
||||
runner->MutableInputs()
|
||||
->Tag("DETECTIONS")
|
||||
->Tag("SALIENT_REGIONS")
|
||||
.packets.push_back(Adopt(detection_set.release()).At(Timestamp(0)));
|
||||
|
||||
// Run the calculator.
|
||||
@@ -166,7 +274,7 @@ TEST(ContentZoomingCalculatorTest, TwoFacesWide) {
|
||||
Adopt(input_frame.release()).At(Timestamp(0)));
|
||||
|
||||
runner->MutableInputs()
|
||||
->Tag("DETECTIONS")
|
||||
->Tag("SALIENT_REGIONS")
|
||||
.packets.push_back(Adopt(detection_set.release()).At(Timestamp(0)));
|
||||
|
||||
// Run the calculator.
|
||||
@@ -191,7 +299,7 @@ TEST(ContentZoomingCalculatorTest, NoDetectionOnInit) {
|
||||
Adopt(input_frame.release()).At(Timestamp(0)));
|
||||
|
||||
runner->MutableInputs()
|
||||
->Tag("DETECTIONS")
|
||||
->Tag("SALIENT_REGIONS")
|
||||
.packets.push_back(Adopt(detection_set.release()).At(Timestamp(0)));
|
||||
|
||||
// Run the calculator.
|
||||
@@ -223,7 +331,7 @@ TEST(ContentZoomingCalculatorTest, ZoomTestPairSize) {
|
||||
.packets.push_back(Adopt(input_size.release()).At(Timestamp(0)));
|
||||
|
||||
runner->MutableInputs()
|
||||
->Tag("DETECTIONS")
|
||||
->Tag("SALIENT_REGIONS")
|
||||
.packets.push_back(Adopt(detection_set.release()).At(Timestamp(0)));
|
||||
|
||||
// Run the calculator.
|
||||
|
||||
@@ -37,7 +37,7 @@ node {
|
||||
output_stream: "TENSORS:detection_tensors"
|
||||
options: {
|
||||
[mediapipe.TfLiteInferenceCalculatorOptions.ext] {
|
||||
model_path: "face_detection_front.tflite"
|
||||
model_path: "mediapipe/models/face_detection_front.tflite"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -118,7 +118,7 @@ node {
|
||||
output_stream: "labeled_detections"
|
||||
options: {
|
||||
[mediapipe.DetectionLabelIdToTextCalculatorOptions.ext] {
|
||||
label_map_path: "face_detection_front_labelmap.txt"
|
||||
label_map_path: "mediapipe/models/face_detection_front_labelmap.txt"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,18 +1 @@
|
||||
This directory contains example MediaPipe applications on iOS.
|
||||
|
||||
| Use Case | Directory |
|
||||
|---------------------------------------|:-----------------------------------:|
|
||||
| Edge Detection on GPU | edgedetection |
|
||||
| Face Detection on CPU | facedetectioncpu |
|
||||
| Face Detection on GPU | facedetectiongpu |
|
||||
| Object Detection on CPU | objectdetectioncpu |
|
||||
| Object Detection on GPU | objectdetectiongpu |
|
||||
| Hand Detection on GPU | handdetectiongpu |
|
||||
| Hand Tracking on GPU | handtrackinggpu |
|
||||
|
||||
For instance, to build an example app for face detection on CPU, run:
|
||||
|
||||
```bash
|
||||
bazel build -c opt --config=ios_arm64 --xcode_version=$XCODE_VERSION --cxxopt='-std=c++14' mediapipe/examples/ios/facedetectioncpu:FaceDetectionCpuApp
|
||||
```
|
||||
(Note: with your own $XCODE_VERSION)
|
||||
This directory contains MediaPipe example applications for iOS. Please see [Solutions](https://solutions.mediapipe.dev)for details.
|
||||
|
||||
@@ -21,6 +21,11 @@ load(
|
||||
"ios_application",
|
||||
)
|
||||
|
||||
alias(
|
||||
name = "edgedetectiongpu",
|
||||
actual = "EdgeDetectionGpuApp",
|
||||
)
|
||||
|
||||
ios_application(
|
||||
name = "EdgeDetectionGpuApp",
|
||||
bundle_id = "com.google.mediapipe.EdgeDetectionGpu",
|
||||
|
||||
@@ -21,6 +21,11 @@ load(
|
||||
"ios_application",
|
||||
)
|
||||
|
||||
alias(
|
||||
name = "facedetectioncpu",
|
||||
actual = "FaceDetectionCpuApp",
|
||||
)
|
||||
|
||||
ios_application(
|
||||
name = "FaceDetectionCpuApp",
|
||||
bundle_id = "com.google.mediapipe.FaceDetectionCpu",
|
||||
|
||||
@@ -21,6 +21,11 @@ load(
|
||||
"ios_application",
|
||||
)
|
||||
|
||||
alias(
|
||||
name = "facedetectiongpu",
|
||||
actual = "FaceDetectionGpuApp",
|
||||
)
|
||||
|
||||
ios_application(
|
||||
name = "FaceDetectionGpuApp",
|
||||
bundle_id = "com.google.mediapipe.FaceDetectionGpu",
|
||||
|
||||
@@ -21,6 +21,11 @@ licenses(["notice"]) # Apache 2.0
|
||||
|
||||
MIN_IOS_VERSION = "10.0"
|
||||
|
||||
alias(
|
||||
name = "facemeshgpu",
|
||||
actual = "FaceMeshGpuApp",
|
||||
)
|
||||
|
||||
ios_application(
|
||||
name = "FaceMeshGpuApp",
|
||||
bundle_id = "com.google.mediapipe.FaceMeshGpu",
|
||||
|
||||
@@ -21,6 +21,11 @@ load(
|
||||
"ios_application",
|
||||
)
|
||||
|
||||
alias(
|
||||
name = "handdetectiongpu",
|
||||
actual = "HandDetectionGpuApp",
|
||||
)
|
||||
|
||||
ios_application(
|
||||
name = "HandDetectionGpuApp",
|
||||
bundle_id = "com.google.mediapipe.HandDetectionGpu",
|
||||
|
||||
@@ -21,6 +21,11 @@ licenses(["notice"]) # Apache 2.0
|
||||
|
||||
MIN_IOS_VERSION = "10.0"
|
||||
|
||||
alias(
|
||||
name = "handtrackinggpu",
|
||||
actual = "HandTrackingGpuApp",
|
||||
)
|
||||
|
||||
ios_application(
|
||||
name = "HandTrackingGpuApp",
|
||||
bundle_id = "com.google.mediapipe.HandTrackingGpu",
|
||||
|
||||
@@ -21,6 +21,11 @@ licenses(["notice"]) # Apache 2.0
|
||||
|
||||
MIN_IOS_VERSION = "10.0"
|
||||
|
||||
alias(
|
||||
name = "multihandtrackinggpu",
|
||||
actual = "MultiHandTrackingGpuApp",
|
||||
)
|
||||
|
||||
ios_application(
|
||||
name = "MultiHandTrackingGpuApp",
|
||||
bundle_id = "com.google.mediapipe.MultiHandTrackingGpu",
|
||||
|
||||
@@ -21,6 +21,11 @@ load(
|
||||
"ios_application",
|
||||
)
|
||||
|
||||
alias(
|
||||
name = "objectdetectioncpu",
|
||||
actual = "ObjectDetectionCpuApp",
|
||||
)
|
||||
|
||||
ios_application(
|
||||
name = "ObjectDetectionCpuApp",
|
||||
bundle_id = "com.google.mediapipe.ObjectDetectionCpu",
|
||||
|
||||
@@ -13,6 +13,7 @@
|
||||
// limitations under the License.
|
||||
|
||||
#import "AppDelegate.h"
|
||||
#import "ViewController.h"
|
||||
|
||||
@interface AppDelegate ()
|
||||
|
||||
@@ -22,7 +23,14 @@
|
||||
|
||||
- (BOOL)application:(UIApplication *)application
|
||||
didFinishLaunchingWithOptions:(NSDictionary *)launchOptions {
|
||||
// Override point for customization after application launch.
|
||||
ViewController *viewController = (ViewController *)self.window.rootViewController;
|
||||
NSURL *url = [launchOptions objectForKey:UIApplicationLaunchOptionsURLKey];
|
||||
// Unattended testing on Firebase is enabled by custom URL schema.
|
||||
if ([url.scheme isEqualToString:@"firebase-game-loop"]) {
|
||||
[viewController setSourceMode:MediaPipeDemoSourceVideo];
|
||||
} else {
|
||||
[viewController setSourceMode:MediaPipeDemoSourceBackCamera];
|
||||
}
|
||||
return YES;
|
||||
}
|
||||
|
||||
|
||||
@@ -21,6 +21,11 @@ load(
|
||||
"ios_application",
|
||||
)
|
||||
|
||||
alias(
|
||||
name = "objectdetectiongpu",
|
||||
actual = "ObjectDetectionGpuApp",
|
||||
)
|
||||
|
||||
ios_application(
|
||||
name = "ObjectDetectionGpuApp",
|
||||
bundle_id = "com.google.mediapipe.ObjectDetectionGpu",
|
||||
|
||||
@@ -38,5 +38,18 @@
|
||||
<array>
|
||||
<string>UIInterfaceOrientationPortrait</string>
|
||||
</array>
|
||||
<key>CFBundleURLTypes</key>
|
||||
<array>
|
||||
<dict>
|
||||
<key>CFBundleURLName</key>
|
||||
<string>com.google.firebase</string>
|
||||
<key>CFBundleTypeRole</key>
|
||||
<string>Editor</string>
|
||||
<key>CFBundleURLSchemes</key>
|
||||
<array>
|
||||
<string>firebase-game-loop</string>
|
||||
</array>
|
||||
</dict>
|
||||
</array>
|
||||
</dict>
|
||||
</plist>
|
||||
|
||||
@@ -14,6 +14,11 @@
|
||||
|
||||
#import <UIKit/UIKit.h>
|
||||
|
||||
@interface ViewController : UIViewController
|
||||
typedef NS_ENUM(NSInteger, MediaPipeDemoSourceMode) {
|
||||
MediaPipeDemoSourceBackCamera,
|
||||
MediaPipeDemoSourceVideo
|
||||
};
|
||||
|
||||
@interface ViewController : UIViewController
|
||||
- (void)setSourceMode:(MediaPipeDemoSourceMode)mode;
|
||||
@end
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
#import "mediapipe/objc/MPPGraph.h"
|
||||
#import "mediapipe/objc/MPPCameraInputSource.h"
|
||||
#import "mediapipe/objc/MPPLayerRenderer.h"
|
||||
#import "mediapipe/objc/MPPPlayerInputSource.h"
|
||||
|
||||
static NSString* const kGraphName = @"mobile_gpu";
|
||||
|
||||
@@ -35,6 +36,8 @@ static const char* kVideoQueueLabel = "com.google.mediapipe.example.videoQueue";
|
||||
@implementation ViewController {
|
||||
/// Handles camera access via AVCaptureSession library.
|
||||
MPPCameraInputSource* _cameraSource;
|
||||
MPPPlayerInputSource* _videoSource;
|
||||
MediaPipeDemoSourceMode _sourceMode;
|
||||
|
||||
/// Inform the user when camera is unavailable.
|
||||
IBOutlet UILabel* _noCameraLabel;
|
||||
@@ -47,6 +50,10 @@ static const char* kVideoQueueLabel = "com.google.mediapipe.example.videoQueue";
|
||||
dispatch_queue_t _videoQueue;
|
||||
}
|
||||
|
||||
- (void)setSourceMode:(MediaPipeDemoSourceMode)mode {
|
||||
_sourceMode = mode;
|
||||
}
|
||||
|
||||
#pragma mark - Cleanup methods
|
||||
|
||||
- (void)dealloc {
|
||||
@@ -97,13 +104,6 @@ static const char* kVideoQueueLabel = "com.google.mediapipe.example.videoQueue";
|
||||
DISPATCH_QUEUE_SERIAL, QOS_CLASS_USER_INTERACTIVE, /*relative_priority=*/0);
|
||||
_videoQueue = dispatch_queue_create(kVideoQueueLabel, qosAttribute);
|
||||
|
||||
_cameraSource = [[MPPCameraInputSource alloc] init];
|
||||
[_cameraSource setDelegate:self queue:_videoQueue];
|
||||
_cameraSource.sessionPreset = AVCaptureSessionPresetHigh;
|
||||
_cameraSource.cameraPosition = AVCaptureDevicePositionBack;
|
||||
// The frame's native format is rotated with respect to the portrait orientation.
|
||||
_cameraSource.orientation = AVCaptureVideoOrientationPortrait;
|
||||
|
||||
self.mediapipeGraph = [[self class] loadGraphFromResource:kGraphName];
|
||||
self.mediapipeGraph.delegate = self;
|
||||
// Set maxFramesInFlight to a small value to avoid memory contention for real-time processing.
|
||||
@@ -119,27 +119,43 @@ static const char* kVideoQueueLabel = "com.google.mediapipe.example.videoQueue";
|
||||
- (void)viewWillAppear:(BOOL)animated {
|
||||
[super viewWillAppear:animated];
|
||||
|
||||
[_cameraSource requestCameraAccessWithCompletionHandler:^void(BOOL granted) {
|
||||
if (granted) {
|
||||
[self startGraphAndCamera];
|
||||
dispatch_async(dispatch_get_main_queue(), ^{
|
||||
_noCameraLabel.hidden = YES;
|
||||
});
|
||||
}
|
||||
}];
|
||||
}
|
||||
|
||||
- (void)startGraphAndCamera {
|
||||
// Start running self.mediapipeGraph.
|
||||
NSError* error;
|
||||
if (![self.mediapipeGraph startWithError:&error]) {
|
||||
NSLog(@"Failed to start graph: %@", error);
|
||||
}
|
||||
|
||||
// Start fetching frames from the camera.
|
||||
dispatch_async(_videoQueue, ^{
|
||||
[_cameraSource start];
|
||||
});
|
||||
switch (_sourceMode) {
|
||||
case MediaPipeDemoSourceVideo: {
|
||||
AVAsset* video =
|
||||
[AVAsset assetWithURL:[[NSBundle mainBundle] URLForResource:@"object_detection"
|
||||
withExtension:@"mov"]];
|
||||
_videoSource = [[MPPPlayerInputSource alloc] initWithAVAsset:video];
|
||||
[_videoSource setDelegate:self queue:_videoQueue];
|
||||
dispatch_async(_videoQueue, ^{
|
||||
[_videoSource start];
|
||||
});
|
||||
break;
|
||||
}
|
||||
case MediaPipeDemoSourceBackCamera:
|
||||
_cameraSource = [[MPPCameraInputSource alloc] init];
|
||||
[_cameraSource setDelegate:self queue:_videoQueue];
|
||||
_cameraSource.sessionPreset = AVCaptureSessionPresetHigh;
|
||||
_cameraSource.cameraPosition = AVCaptureDevicePositionBack;
|
||||
// The frame's native format is rotated with respect to the portrait orientation.
|
||||
_cameraSource.orientation = AVCaptureVideoOrientationPortrait;
|
||||
[_cameraSource requestCameraAccessWithCompletionHandler:^void(BOOL granted) {
|
||||
if (granted) {
|
||||
dispatch_async(_videoQueue, ^{
|
||||
[_cameraSource start];
|
||||
});
|
||||
dispatch_async(dispatch_get_main_queue(), ^{
|
||||
_noCameraLabel.hidden = YES;
|
||||
});
|
||||
}
|
||||
}];
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
#pragma mark - MPPGraphDelegate methods
|
||||
@@ -164,7 +180,7 @@ static const char* kVideoQueueLabel = "com.google.mediapipe.example.videoQueue";
|
||||
- (void)processVideoFrame:(CVPixelBufferRef)imageBuffer
|
||||
timestamp:(CMTime)timestamp
|
||||
fromSource:(MPPInputSource*)source {
|
||||
if (source != _cameraSource) {
|
||||
if (source != _cameraSource && source != _videoSource) {
|
||||
NSLog(@"Unknown source: %@", source);
|
||||
return;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user