talksik/mediapipe
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Project import generated by Copybara.

Browse Source 
GitOrigin-RevId: 4cee4a2c2317fb190680c17e31ebbb03bb73b71c
This commit is contained in:
MediaPipe Team
2020-09-17 11:09:17 -04:00
committed by chuoling
parent 1db91b550a
commit a908d668c7
142 changed files with 40878 additions and 574 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mediapipe/MediaPipe.tulsiproj/Configs/MediaPipe.tulsigen
+5
View File
@@ -10,6 +10,7 @@
"mediapipe/examples/ios/helloworld/BUILD",
"mediapipe/examples/ios/facedetectioncpu/BUILD",
"mediapipe/examples/ios/facedetectiongpu/BUILD",
"mediapipe/examples/ios/faceeffect/BUILD",
"mediapipe/examples/ios/facemeshgpu/BUILD",
"mediapipe/examples/ios/handdetectiongpu/BUILD",
"mediapipe/examples/ios/handtrackinggpu/BUILD",
@@ -23,6 +24,7 @@
"//mediapipe/examples/ios/helloworld:HelloWorldApp",
"//mediapipe/examples/ios/facedetectioncpu:FaceDetectionCpuApp",
"//mediapipe/examples/ios/facedetectiongpu:FaceDetectionGpuApp",
"//mediapipe/examples/ios/faceeffect:FaceEffectApp",
"//mediapipe/examples/ios/facemeshgpu:FaceMeshGpuApp",
"//mediapipe/examples/ios/handdetectiongpu:HandDetectionGpuApp",
"//mediapipe/examples/ios/handtrackinggpu:HandTrackingGpuApp",
@@ -90,6 +92,8 @@
"mediapipe/examples/ios/helloworld",
"mediapipe/examples/ios/facedetectioncpu",
"mediapipe/examples/ios/facedetectiongpu",
"mediapipe/examples/ios/faceeffect",
"mediapipe/examples/ios/faceeffect/Base.lproj",
"mediapipe/examples/ios/handdetectiongpu",
"mediapipe/examples/ios/handtrackinggpu",
"mediapipe/examples/ios/iristrackinggpu",
@@ -110,6 +114,7 @@
"mediapipe/graphs",
"mediapipe/graphs/edge_detection",
"mediapipe/graphs/face_detection",
"mediapipe/graphs/face_geometry",
"mediapipe/graphs/hand_tracking",
"mediapipe/graphs/object_detection",
"mediapipe/graphs/pose_tracking",
mediapipe/MediaPipe.tulsiproj/project.tulsiconf
+1
View File
@@ -13,6 +13,7 @@
"mediapipe/examples/ios",
"mediapipe/examples/ios/facedetectioncpu",
"mediapipe/examples/ios/facedetectiongpu",
"mediapipe/examples/ios/faceeffect",
"mediapipe/examples/ios/facemeshgpu",
"mediapipe/examples/ios/handdetectiongpu",
"mediapipe/examples/ios/handtrackinggpu",
mediapipe/calculators/core/concatenate_vector_calculator.cc
+3
View File
@@ -50,6 +50,9 @@ REGISTER_CALCULATOR(ConcatenateInt32VectorCalculator);
typedef ConcatenateVectorCalculator<uint64> ConcatenateUInt64VectorCalculator;
REGISTER_CALCULATOR(ConcatenateUInt64VectorCalculator);
typedef ConcatenateVectorCalculator<bool> ConcatenateBoolVectorCalculator;
REGISTER_CALCULATOR(ConcatenateBoolVectorCalculator);
// Example config:
// node {
// calculator: "ConcatenateTfLiteTensorVectorCalculator"
mediapipe/calculators/core/mux_calculator_test.cc
+68
View File
@@ -12,6 +12,8 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include <memory>
#include "mediapipe/calculators/core/split_vector_calculator.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_runner.h"
@@ -233,5 +235,71 @@ TEST(MuxCalculatorTest, InputStreamSelector_MuxInputStreamHandler) {
kOutputName, output_fn);
EXPECT_EQ(output, input_packets);
}
constexpr char kDualInputGraphConfig[] = R"proto(
input_stream: "input_0"
input_stream: "input_1"
input_stream: "input_select"
output_stream: "test_output"
node {
calculator: "MuxCalculator"
input_stream: "INPUT:0:input_0"
input_stream: "INPUT:1:input_1"
input_stream: "SELECT:input_select"
output_stream: "OUTPUT:test_output"
}
)proto";
TEST(MuxCalculatorTest, DiscardSkippedInputs_MuxInputStreamHandler) {
CalculatorGraphConfig config =
::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
kDualInputGraphConfig);
CalculatorGraph graph;
MP_ASSERT_OK(graph.Initialize(config));
std::shared_ptr<int> output;
MP_ASSERT_OK(
graph.ObserveOutputStream("test_output", [&output](const Packet& p) {
output = p.Get<std::shared_ptr<int>>();
return ::mediapipe::OkStatus();
}));
MP_ASSERT_OK(graph.StartRun({}));
auto one = std::make_shared<int>(1);
auto two = std::make_shared<int>(2);
auto three = std::make_shared<int>(3);
std::weak_ptr<int> one_weak = one;
std::weak_ptr<int> two_weak = two;
MP_ASSERT_OK(graph.AddPacketToInputStream(
"input_0",
MakePacket<std::shared_ptr<int>>(std::move(one)).At(Timestamp(0))));
MP_ASSERT_OK(graph.AddPacketToInputStream(
"input_1",
MakePacket<std::shared_ptr<int>>(std::move(two)).At(Timestamp(0))));
MP_ASSERT_OK(graph.AddPacketToInputStream(
"input_1",
MakePacket<std::shared_ptr<int>>(std::move(three)).At(Timestamp(1))));
EXPECT_EQ(one, nullptr);
EXPECT_EQ(two, nullptr);
EXPECT_EQ(three, nullptr);
MP_ASSERT_OK(graph.AddPacketToInputStream(
"input_select", MakePacket<int>(0).At(Timestamp(0))));
MP_ASSERT_OK(graph.WaitUntilIdle());
EXPECT_EQ(*output, 1);
EXPECT_NE(one_weak.lock(), nullptr);
EXPECT_EQ(two_weak.lock(), nullptr);
MP_ASSERT_OK(graph.AddPacketToInputStream(
"input_select", MakePacket<int>(1).At(Timestamp(1))));
MP_ASSERT_OK(graph.WaitUntilIdle());
EXPECT_EQ(*output, 3);
MP_ASSERT_OK(graph.CloseAllInputStreams());
MP_ASSERT_OK(graph.WaitUntilDone());
}
} // namespace
} // namespace mediapipe
mediapipe/calculators/util/BUILD
+27 -1
View File
@@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library", "mediapipe_proto_library")
licenses(["notice"])
@@ -945,6 +945,31 @@ cc_library(
alwayslink = 1,
)
mediapipe_proto_library(
name = "landmarks_smoothing_calculator_proto",
srcs = ["landmarks_smoothing_calculator.proto"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_options_proto",
],
)
cc_library(
name = "landmarks_smoothing_calculator",
srcs = ["landmarks_smoothing_calculator.cc"],
visibility = ["//visibility:public"],
deps = [
":landmarks_smoothing_calculator_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:timestamp",
"//mediapipe/framework/formats:landmark_cc_proto",
"//mediapipe/framework/port:ret_check",
"//mediapipe/util/filtering:relative_velocity_filter",
"@com_google_absl//absl/algorithm:container",
],
alwayslink = 1,
)
cc_library(
name = "landmarks_to_floats_calculator",
srcs = ["landmarks_to_floats_calculator.cc"],
@@ -1103,6 +1128,7 @@ cc_library(
"//mediapipe/framework/formats:classification_cc_proto",
"//mediapipe/framework/formats:landmark_cc_proto",
"//mediapipe/framework/formats:rect_cc_proto",
"//mediapipe/framework/port:integral_types",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"@com_google_absl//absl/strings",
mediapipe/calculators/util/filter_collection_calculator.cc
+5
View File
@@ -20,9 +20,14 @@
#include "mediapipe/framework/formats/classification.pb.h"
#include "mediapipe/framework/formats/landmark.pb.h"
#include "mediapipe/framework/formats/rect.pb.h"
#include "mediapipe/framework/port/integral_types.h"
namespace mediapipe {
typedef FilterCollectionCalculator<std::vector<uint64>>
FilterUInt64CollectionCalculator;
REGISTER_CALCULATOR(FilterUInt64CollectionCalculator);
typedef FilterCollectionCalculator<std::vector<::mediapipe::NormalizedRect>>
FilterNormalizedRectCollectionCalculator;
REGISTER_CALCULATOR(FilterNormalizedRectCollectionCalculator);
mediapipe/graphs/pose_tracking/calculators/landmarks_smoothing_calculator.cc → mediapipe/calculators/util/landmarks_smoothing_calculator.cc
+10 -10
View File
@@ -13,12 +13,12 @@
// limitations under the License.
#include "absl/algorithm/container.h"
#include "mediapipe/calculators/util/landmarks_smoothing_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/landmark.pb.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/timestamp.h"
#include "mediapipe/graphs/pose_tracking/calculators/landmarks_smoothing_calculator.pb.h"
#include "mediapipe/graphs/pose_tracking/calculators/relative_velocity_filter.h"
#include "mediapipe/util/filtering/relative_velocity_filter.h"
namespace mediapipe {
@@ -38,17 +38,17 @@ using ::mediapipe::RelativeVelocityFilter;
// with sides parallel to axis.
float GetObjectScale(const NormalizedLandmarkList& landmarks, int image_width,
int image_height) {
const auto& [lm_min_x, lm_max_x] = absl::c_minmax_element(
const auto& lm_minmax_x = absl::c_minmax_element(
landmarks.landmark(),
[](const auto& a, const auto& b) { return a.x() < b.x(); });
const float x_min = lm_min_x->x();
const float x_max = lm_max_x->x();
const float x_min = lm_minmax_x.first->x();
const float x_max = lm_minmax_x.second->x();
const auto& [lm_min_y, lm_max_y] = absl::c_minmax_element(
const auto& lm_minmax_y = absl::c_minmax_element(
landmarks.landmark(),
[](const auto& a, const auto& b) { return a.y() < b.y(); });
const float y_min = lm_min_y->y();
const float y_max = lm_max_y->y();
const float y_min = lm_minmax_y.first->y();
const float y_max = lm_minmax_y.second->y();
const float object_width = (x_max - x_min) * image_width;
const float object_height = (y_max - y_min) * image_height;
@@ -191,8 +191,8 @@ class VelocityFilter : public LandmarksFilter {
// input_stream: "NORM_LANDMARKS:pose_landmarks"
// input_stream: "IMAGE_SIZE:image_size"
// output_stream: "NORM_FILTERED_LANDMARKS:pose_landmarks_filtered"
// node_options: {
// [type.googleapis.com/mediapipe.LandmarksSmoothingCalculatorOptions] {
// options: {
// [mediapipe.LandmarksSmoothingCalculatorOptions.ext] {
// velocity_filter: {
// window_size: 5
// velocity_scale: 10.0
mediapipe/graphs/pose_tracking/calculators/landmarks_smoothing_calculator.proto → mediapipe/calculators/util/landmarks_smoothing_calculator.proto
+1 -1
View File
@@ -16,7 +16,7 @@ syntax = "proto2";
package mediapipe;
import "mediapipe/framework/calculator.proto";
import "mediapipe/framework/calculator_options.proto";
message LandmarksSmoothingCalculatorOptions {
extend CalculatorOptions {
mediapipe/calculators/util/landmarks_to_render_data_calculator.cc
+4 -4
View File
@@ -103,8 +103,8 @@ void AddConnectionsWithDepth(const LandmarkListType& landmarks,
for (int i = 0; i < landmark_connections.size(); i += 2) {
const auto& ld0 = landmarks.landmark(landmark_connections[i]);
const auto& ld1 = landmarks.landmark(landmark_connections[i + 1]);
if (visibility_threshold && (ld0.visibility() < visibility_threshold ||
ld1.visibility() < visibility_threshold)) {
if (utilize_visibility && (ld0.visibility() < visibility_threshold ||
ld1.visibility() < visibility_threshold)) {
continue;
}
const int gray_val1 =
@@ -141,8 +141,8 @@ void AddConnections(const LandmarkListType& landmarks,
for (int i = 0; i < landmark_connections.size(); i += 2) {
const auto& ld0 = landmarks.landmark(landmark_connections[i]);
const auto& ld1 = landmarks.landmark(landmark_connections[i + 1]);
if (visibility_threshold && (ld0.visibility() < visibility_threshold ||
ld1.visibility() < visibility_threshold)) {
if (utilize_visibility && (ld0.visibility() < visibility_threshold ||
ld1.visibility() < visibility_threshold)) {
continue;
}
AddConnectionToRenderData<LandmarkType>(ld0, ld1, connection_color,
mediapipe/examples/android/src/java/com/google/mediapipe/apps/faceeffect/BUILD
+67
View File
@@ -0,0 +1,67 @@
# Copyright 2020 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
licenses(["notice"])
package(default_visibility = ["//visibility:private"])
cc_binary(
name = "libmediapipe_jni.so",
linkshared = 1,
linkstatic = 1,
deps = [
"//mediapipe/graphs/face_effect:face_effect_gpu_deps",
"//mediapipe/java/com/google/mediapipe/framework/jni:mediapipe_framework_jni",
],
)
cc_library(
name = "mediapipe_jni_lib",
srcs = [":libmediapipe_jni.so"],
alwayslink = 1,
)
android_binary(
name = "faceeffect",
srcs = glob(["*.java"]),
assets = [
"//mediapipe/graphs/face_effect/data:facepaint.pngblob",
"//mediapipe/graphs/face_effect/data:glasses.binarypb",
"//mediapipe/graphs/face_effect/data:glasses.pngblob",
"//mediapipe/graphs/face_effect:face_effect_gpu.binarypb",
"//mediapipe/modules/face_detection:face_detection_front.tflite",
"//mediapipe/modules/face_geometry/data:geometry_pipeline_metadata.binarypb",
"//mediapipe/modules/face_landmark:face_landmark.tflite",
],
assets_dir = "",
manifest = "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:AndroidManifest.xml",
manifest_values = {
"applicationId": "com.google.mediapipe.apps.faceeffect",
"appName": "Face Effect",
"mainActivity": ".MainActivity",
"cameraFacingFront": "True",
"binaryGraphName": "face_effect_gpu.binarypb",
"inputVideoStreamName": "input_video",
"outputVideoStreamName": "output_video",
"flipFramesVertically": "True",
},
multidex = "native",
deps = [
":mediapipe_jni_lib",
"//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:basic_lib",
"//mediapipe/framework/formats:matrix_data_java_proto_lite",
"//mediapipe/java/com/google/mediapipe/framework:android_framework",
"//mediapipe/modules/face_geometry/protos:face_geometry_java_proto_lite",
],
)
mediapipe/examples/android/src/java/com/google/mediapipe/apps/faceeffect/MainActivity.java
+176
View File
@@ -0,0 +1,176 @@
// Copyright 2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.mediapipe.apps.faceeffect;
import android.graphics.Color;
import android.os.Bundle;
import android.util.Log;
import android.view.GestureDetector;
import android.view.Gravity;
import android.view.MotionEvent;
import android.view.View;
import android.view.ViewGroup;
import android.view.ViewGroup.LayoutParams;
import android.widget.RelativeLayout;
import android.widget.TextView;
import com.google.mediapipe.framework.Packet;
import com.google.mediapipe.framework.PacketGetter;
import com.google.mediapipe.modules.facegeometry.FaceGeometryProto.FaceGeometry;
import com.google.mediapipe.formats.proto.MatrixDataProto.MatrixData;
import java.util.List;
/** Main activity of MediaPipe face mesh app. */
public class MainActivity extends com.google.mediapipe.apps.basic.MainActivity {
private static final String TAG = "MainActivity";
// Stream names.
private static final String IS_FACEPAINT_EFFECT_SELECTED_INPUT_STREAM_NAME =
"is_facepaint_effect_selected";
private static final String OUTPUT_FACE_GEOMETRY_STREAM_NAME = "multi_face_geometry";
private static final String EFFECT_SWITCHING_HINT_TEXT = "Tap to switch between effects!";
private static final int MATRIX_TRANSLATION_Z_INDEX = 14;
private final Object isFacepaintEffectSelectedLock = new Object();
private boolean isFacepaintEffectSelected;
private View effectSwitchingHintView;
private GestureDetector tapGestureDetector;
@Override
protected void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
// Add an effect switching hint view to the preview layout.
effectSwitchingHintView = createEffectSwitchingHintView();
effectSwitchingHintView.setVisibility(View.INVISIBLE);
ViewGroup viewGroup = findViewById(R.id.preview_display_layout);
viewGroup.addView(effectSwitchingHintView);
// By default, render the glasses effect.
isFacepaintEffectSelected = false;
// This callback demonstrates how the output face geometry packet can be obtained and used
// in an Android app. As an example, the Z-translation component of the face pose transform
// matrix is logged for each face being equal to the approximate distance away from the camera
// in centimeters.
processor.addPacketCallback(
OUTPUT_FACE_GEOMETRY_STREAM_NAME,
(packet) -> {
effectSwitchingHintView.post(
new Runnable() {
@Override
public void run() {
effectSwitchingHintView.setVisibility(View.VISIBLE);
}
});
Log.d(TAG, "Received a multi face geometry packet.");
List<FaceGeometry> multiFaceGeometry =
PacketGetter.getProtoVector(packet, FaceGeometry.parser());
StringBuilder approxDistanceAwayFromCameraLogMessage = new StringBuilder();
for (FaceGeometry faceGeometry : multiFaceGeometry) {
if (approxDistanceAwayFromCameraLogMessage.length() > 0) {
approxDistanceAwayFromCameraLogMessage.append(' ');
}
MatrixData poseTransformMatrix = faceGeometry.getPoseTransformMatrix();
approxDistanceAwayFromCameraLogMessage.append(
-poseTransformMatrix.getPackedData(MATRIX_TRANSLATION_Z_INDEX));
}
Log.d(
TAG,
"[TS:"
+ packet.getTimestamp()
+ "] size = "
+ multiFaceGeometry.size()
+ "; approx. distance away from camera in cm for faces = ["
+ approxDistanceAwayFromCameraLogMessage
+ "]");
});
// Alongside the input camera frame, we also send the `is_facepaint_effect_selected` boolean
// packet to indicate which effect should be rendered on this frame.
processor.setOnWillAddFrameListener(
(timestamp) -> {
Packet isFacepaintEffectSelectedPacket = null;
try {
synchronized (isFacepaintEffectSelectedLock) {
isFacepaintEffectSelectedPacket =
processor.getPacketCreator().createBool(isFacepaintEffectSelected);
}
processor
.getGraph()
.addPacketToInputStream(
IS_FACEPAINT_EFFECT_SELECTED_INPUT_STREAM_NAME,
isFacepaintEffectSelectedPacket,
timestamp);
} catch (RuntimeException e) {
Log.e(
TAG,
"Exception while adding packet to input stream while switching effects: " + e);
} finally {
if (isFacepaintEffectSelectedPacket != null) {
isFacepaintEffectSelectedPacket.release();
}
}
});
// We use the tap gesture detector to switch between face effects. This allows users to try
// multiple pre-bundled face effects without a need to recompile the app.
tapGestureDetector =
new GestureDetector(
this,
new GestureDetector.SimpleOnGestureListener() {
@Override
public void onLongPress(MotionEvent event) {
switchEffect();
}
@Override
public boolean onSingleTapUp(MotionEvent event) {
switchEffect();
return true;
}
private void switchEffect() {
synchronized (isFacepaintEffectSelectedLock) {
isFacepaintEffectSelected = !isFacepaintEffectSelected;
}
}
});
}
@Override
public boolean onTouchEvent(MotionEvent event) {
return tapGestureDetector.onTouchEvent(event);
}
private View createEffectSwitchingHintView() {
TextView effectSwitchingHintView = new TextView(getApplicationContext());
effectSwitchingHintView.setLayoutParams(
new RelativeLayout.LayoutParams(LayoutParams.FILL_PARENT, LayoutParams.FILL_PARENT));
effectSwitchingHintView.setText(EFFECT_SWITCHING_HINT_TEXT);
effectSwitchingHintView.setGravity(Gravity.CENTER_HORIZONTAL | Gravity.BOTTOM);
effectSwitchingHintView.setPadding(0, 0, 0, 480);
effectSwitchingHintView.setTextColor(Color.parseColor("#ffffff"));
effectSwitchingHintView.setTextSize((float) 24);
return effectSwitchingHintView;
}
}
mediapipe/examples/desktop/autoflip/calculators/content_zooming_calculator.cc
+17 -8
View File
@@ -35,7 +35,7 @@ constexpr char kDetectedBorders[] = "BORDERS";
constexpr char kCropRect[] = "CROP_RECT";
// Field-of-view (degrees) of the camera's x-axis (width).
// TODO: Parameterize FOV based on camera specs.
constexpr float kWidthFieldOfView = 60;
constexpr float kFieldOfView = 60;
namespace mediapipe {
namespace autoflip {
@@ -256,13 +256,13 @@ void MakeStaticFeatures(const int top_border, const int bottom_border,
if (!initialized_) {
path_solver_height_ = std::make_unique<KinematicPathSolver>(
options_.kinematic_options_zoom(), 0, frame_height_,
static_cast<float>(frame_width_) / kWidthFieldOfView);
static_cast<float>(frame_height_) / kFieldOfView);
path_solver_width_ = std::make_unique<KinematicPathSolver>(
options_.kinematic_options_pan(), 0, frame_width_,
static_cast<float>(frame_width_) / kWidthFieldOfView);
static_cast<float>(frame_width_) / kFieldOfView);
path_solver_offset_ = std::make_unique<KinematicPathSolver>(
options_.kinematic_options_tilt(), 0, frame_height_,
static_cast<float>(frame_width_) / kWidthFieldOfView);
static_cast<float>(frame_height_) / kFieldOfView);
max_frame_value_ = 1.0;
target_aspect_ = frame_width_ / static_cast<float>(frame_height_);
// If target size is set and wider than input aspect, make sure to always
@@ -339,15 +339,24 @@ void MakeStaticFeatures(const int top_border, const int bottom_border,
offset_y = last_measured_y_offset_;
}
// Compute smoothed camera paths.
// Compute smoothed zoom camera path.
MP_RETURN_IF_ERROR(path_solver_height_->AddObservation(
height, cc->InputTimestamp().Microseconds()));
int path_height;
MP_RETURN_IF_ERROR(path_solver_height_->GetState(&path_height));
int path_width = path_height * target_aspect_;
// Update pixel-per-degree value for pan/tilt.
MP_RETURN_IF_ERROR(path_solver_width_->UpdatePixelsPerDegree(
static_cast<float>(path_width) / kFieldOfView));
MP_RETURN_IF_ERROR(path_solver_offset_->UpdatePixelsPerDegree(
static_cast<float>(path_height) / kFieldOfView));
// Compute smoothed pan/tilt paths.
MP_RETURN_IF_ERROR(path_solver_width_->AddObservation(
offset_x, cc->InputTimestamp().Microseconds()));
MP_RETURN_IF_ERROR(path_solver_offset_->AddObservation(
offset_y, cc->InputTimestamp().Microseconds()));
int path_height;
MP_RETURN_IF_ERROR(path_solver_height_->GetState(&path_height));
int path_offset_x;
MP_RETURN_IF_ERROR(path_solver_width_->GetState(&path_offset_x));
int path_offset_y;
@@ -359,7 +368,7 @@ void MakeStaticFeatures(const int top_border, const int bottom_border,
} else if (path_offset_y + ceil(path_height / 2.0) > frame_height_) {
path_offset_y = frame_height_ - ceil(path_height / 2.0);
}
int path_width = path_height * target_aspect_;
if (path_offset_x - ceil(path_width / 2.0) < 0) {
path_offset_x = ceil(path_width / 2.0);
} else if (path_offset_x + ceil(path_width / 2.0) > frame_width_) {
mediapipe/examples/desktop/autoflip/calculators/content_zooming_calculator_test.cc
+8 -8
View File
@@ -174,15 +174,15 @@ TEST(ContentZoomingCalculatorTest, PanConfig) {
ContentZoomingCalculatorOptions::ext);
options->mutable_kinematic_options_pan()->set_min_motion_to_reframe(0.0);
options->mutable_kinematic_options_pan()->set_update_rate_seconds(2);
options->mutable_kinematic_options_tilt()->set_min_motion_to_reframe(5.0);
options->mutable_kinematic_options_zoom()->set_min_motion_to_reframe(5.0);
options->mutable_kinematic_options_tilt()->set_min_motion_to_reframe(50.0);
options->mutable_kinematic_options_zoom()->set_min_motion_to_reframe(50.0);
auto runner = ::absl::make_unique<CalculatorRunner>(config);
AddDetection(cv::Rect_<float>(.4, .5, .1, .1), 0, runner.get());
AddDetection(cv::Rect_<float>(.45, .55, .15, .15), 1000000, runner.get());
MP_ASSERT_OK(runner->Run());
CheckCropRect(450, 550, 111, 111, 0,
runner->Outputs().Tag("CROP_RECT").packets);
CheckCropRect(488, 550, 111, 111, 1,
CheckCropRect(483, 550, 111, 111, 1,
runner->Outputs().Tag("CROP_RECT").packets);
}
@@ -190,17 +190,17 @@ TEST(ContentZoomingCalculatorTest, TiltConfig) {
auto config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfigD);
auto* options = config.mutable_options()->MutableExtension(
ContentZoomingCalculatorOptions::ext);
options->mutable_kinematic_options_pan()->set_min_motion_to_reframe(5.0);
options->mutable_kinematic_options_pan()->set_min_motion_to_reframe(50.0);
options->mutable_kinematic_options_tilt()->set_min_motion_to_reframe(0.0);
options->mutable_kinematic_options_tilt()->set_update_rate_seconds(2);
options->mutable_kinematic_options_zoom()->set_min_motion_to_reframe(5.0);
options->mutable_kinematic_options_zoom()->set_min_motion_to_reframe(50.0);
auto runner = ::absl::make_unique<CalculatorRunner>(config);
AddDetection(cv::Rect_<float>(.4, .5, .1, .1), 0, runner.get());
AddDetection(cv::Rect_<float>(.45, .55, .15, .15), 1000000, runner.get());
MP_ASSERT_OK(runner->Run());
CheckCropRect(450, 550, 111, 111, 0,
runner->Outputs().Tag("CROP_RECT").packets);
CheckCropRect(450, 588, 111, 111, 1,
CheckCropRect(450, 583, 111, 111, 1,
runner->Outputs().Tag("CROP_RECT").packets);
}
@@ -208,8 +208,8 @@ TEST(ContentZoomingCalculatorTest, ZoomConfig) {
auto config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfigD);
auto* options = config.mutable_options()->MutableExtension(
ContentZoomingCalculatorOptions::ext);
options->mutable_kinematic_options_pan()->set_min_motion_to_reframe(5.0);
options->mutable_kinematic_options_tilt()->set_min_motion_to_reframe(5.0);
options->mutable_kinematic_options_pan()->set_min_motion_to_reframe(50.0);
options->mutable_kinematic_options_tilt()->set_min_motion_to_reframe(50.0);
options->mutable_kinematic_options_zoom()->set_min_motion_to_reframe(0.0);
options->mutable_kinematic_options_zoom()->set_update_rate_seconds(2);
auto runner = ::absl::make_unique<CalculatorRunner>(config);
mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver.cc
+8
View File
@@ -87,5 +87,13 @@ namespace autoflip {
return ::mediapipe::OkStatus();
}
::mediapipe::Status KinematicPathSolver::UpdatePixelsPerDegree(
const float pixels_per_degree) {
RET_CHECK_GT(pixels_per_degree_, 0)
<< "pixels_per_degree must be larger than 0.";
pixels_per_degree_ = pixels_per_degree;
return ::mediapipe::OkStatus();
}
} // namespace autoflip
} // namespace mediapipe
mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver.h
+2
View File
@@ -46,6 +46,8 @@ class KinematicPathSolver {
::mediapipe::Status UpdatePrediction(const int64 time_us);
// Get the state at a time.
::mediapipe::Status GetState(int* position);
// Update PixelPerDegree value.
::mediapipe::Status UpdatePixelsPerDegree(const float pixels_per_degree);
private:
// Tuning options.
mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver_test.cc
+22
View File
@@ -207,6 +207,28 @@ TEST(KinematicPathSolverTest, PassMaxVelocity) {
EXPECT_EQ(state, 600);
}
TEST(KinematicPathSolverTest, PassDegPerPxChange) {
KinematicOptions options;
// Set min motion to 2deg
options.set_min_motion_to_reframe(2.0);
options.set_update_rate(1);
options.set_max_velocity(1000);
// Set degrees / pixel to 16.6
KinematicPathSolver solver(options, 0, 1000, 1000.0 / kWidthFieldOfView);
int state;
MP_ASSERT_OK(solver.AddObservation(500, kMicroSecInSec * 0));
// Move target by 20px / 16.6 = 1.2deg
MP_ASSERT_OK(solver.AddObservation(520, kMicroSecInSec * 1));
MP_ASSERT_OK(solver.GetState(&state));
// Expect cam to not move.
EXPECT_EQ(state, 500);
MP_ASSERT_OK(solver.UpdatePixelsPerDegree(500.0 / kWidthFieldOfView));
MP_ASSERT_OK(solver.AddObservation(520, kMicroSecInSec * 2));
MP_ASSERT_OK(solver.GetState(&state));
// Expect cam to move.
EXPECT_EQ(state, 516);
}
} // namespace
} // namespace autoflip
} // namespace mediapipe
mediapipe/examples/desktop/autoflip/subgraph/front_face_detection_subgraph.pbtxt
+3 -3
View File
@@ -1,5 +1,5 @@
# MediaPipe graph that performs face detection with TensorFlow Lite on CPU. Model paths setup for web use.
# TODO: parameterize input paths to support desktop use.
# TODO: parameterize input paths to support desktop use, for web only.
input_stream: "VIDEO:input_video"
output_stream: "DETECTIONS:output_detections"
@@ -37,7 +37,7 @@ node {
output_stream: "TENSORS:detection_tensors"
options: {
[mediapipe.TfLiteInferenceCalculatorOptions.ext] {
model_path: "mediapipe/models/face_detection_front.tflite"
model_path: "face_detection_front.tflite"
}
}
}
@@ -118,7 +118,7 @@ node {
output_stream: "labeled_detections"
options: {
[mediapipe.DetectionLabelIdToTextCalculatorOptions.ext] {
label_map_path: "mediapipe/models/face_detection_front_labelmap.txt"
label_map_path: "face_detection_front_labelmap.txt"
}
}
}
mediapipe/examples/ios/faceeffect/AppDelegate.h
+21
View File
@@ -0,0 +1,21 @@
// Copyright 2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#import <UIKit/UIKit.h>
@interface AppDelegate : UIResponder <UIApplicationDelegate>
@property(strong, nonatomic) UIWindow *window;
@end
mediapipe/examples/ios/faceeffect/AppDelegate.m
+59
View File
@@ -0,0 +1,59 @@
// Copyright 2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#import "AppDelegate.h"
@interface AppDelegate ()
@end
@implementation AppDelegate
- (BOOL)application:(UIApplication *)application
didFinishLaunchingWithOptions:(NSDictionary *)launchOptions {
// Override point for customization after application launch.
return YES;
}
- (void)applicationWillResignActive:(UIApplication *)application {
// Sent when the application is about to move from active to inactive state. This can occur for
// certain types of temporary interruptions (such as an incoming phone call or SMS message) or
// when the user quits the application and it begins the transition to the background state. Use
// this method to pause ongoing tasks, disable timers, and invalidate graphics rendering
// callbacks. Games should use this method to pause the game.
}
- (void)applicationDidEnterBackground:(UIApplication *)application {
// Use this method to release shared resources, save user data, invalidate timers, and store
// enough application state information to restore your application to its current state in case
// it is terminated later. If your application supports background execution, this method is
// called instead of applicationWillTerminate: when the user quits.
}
- (void)applicationWillEnterForeground:(UIApplication *)application {
// Called as part of the transition from the background to the active state; here you can undo
// many of the changes made on entering the background.
}
- (void)applicationDidBecomeActive:(UIApplication *)application {
// Restart any tasks that were paused (or not yet started) while the application was inactive. If
// the application was previously in the background, optionally refresh the user interface.
}
- (void)applicationWillTerminate:(UIApplication *)application {
// Called when the application is about to terminate. Save data if appropriate. See also
// applicationDidEnterBackground:.
}
@end
mediapipe/examples/ios/faceeffect/BUILD
+92
View File
@@ -0,0 +1,92 @@
# Copyright 2020 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
load(
"@build_bazel_rules_apple//apple:ios.bzl",
"ios_application",
)
load(
"//mediapipe/examples/ios:bundle_id.bzl",
"BUNDLE_ID_PREFIX",
"example_provisioning",
)
licenses(["notice"])
MIN_IOS_VERSION = "10.0"
alias(
name = "faceeffect",
actual = "FaceEffectApp",
)
ios_application(
name = "FaceEffectApp",
app_icons = ["//mediapipe/examples/ios/common:AppIcon"],
bundle_id = BUNDLE_ID_PREFIX + ".FaceMeshGpu",
families = [
"iphone",
"ipad",
],
infoplists = ["Info.plist"],
minimum_os_version = MIN_IOS_VERSION,
provisioning_profile = example_provisioning(),
deps = [
":FaceEffectAppLibrary",
"@ios_opencv//:OpencvFramework",
],
)
objc_library(
name = "FaceEffectAppLibrary",
srcs = [
"AppDelegate.m",
"FaceEffectViewController.mm",
"main.m",
],
hdrs = [
"AppDelegate.h",
"FaceEffectViewController.h",
],
data = [
"Base.lproj/LaunchScreen.storyboard",
"Base.lproj/Main.storyboard",
"//mediapipe/graphs/face_effect:face_effect_gpu.binarypb",
"//mediapipe/graphs/face_effect/data:facepaint.pngblob",
"//mediapipe/graphs/face_effect/data:glasses.binarypb",
"//mediapipe/graphs/face_effect/data:glasses.pngblob",
"//mediapipe/modules/face_detection:face_detection_front.tflite",
"//mediapipe/modules/face_geometry/data:geometry_pipeline_metadata.binarypb",
"//mediapipe/modules/face_landmark:face_landmark.tflite",
],
sdk_frameworks = [
"AVFoundation",
"CoreGraphics",
"CoreMedia",
"UIKit",
],
deps = [
"//mediapipe/objc:mediapipe_framework_ios",
"//mediapipe/objc:mediapipe_input_sources_ios",
"//mediapipe/objc:mediapipe_layer_renderer",
] + select({
"//mediapipe:ios_i386": [],
"//mediapipe:ios_x86_64": [],
"//conditions:default": [
"//mediapipe/framework/formats:matrix_data_cc_proto",
"//mediapipe/graphs/face_effect:face_effect_gpu_deps",
"//mediapipe/modules/face_geometry/protos:face_geometry_cc_proto",
],
}),
)
mediapipe/examples/ios/faceeffect/Base.lproj/LaunchScreen.storyboard
+25
View File
@@ -0,0 +1,25 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<document type="com.apple.InterfaceBuilder3.CocoaTouch.Storyboard.XIB" version="3.0" toolsVersion="13122.16" targetRuntime="iOS.CocoaTouch" propertyAccessControl="none" useAutolayout="YES" launchScreen="YES" useTraitCollections="YES" useSafeAreas="YES" colorMatched="YES" initialViewController="01J-lp-oVM">
<dependencies>
<plugIn identifier="com.apple.InterfaceBuilder.IBCocoaTouchPlugin" version="13104.12"/>
<capability name="Safe area layout guides" minToolsVersion="9.0"/>
<capability name="documents saved in the Xcode 8 format" minToolsVersion="8.0"/>
</dependencies>
<scenes>
<!--View Controller-->
<scene sceneID="EHf-IW-A2E">
<objects>
<viewController id="01J-lp-oVM" sceneMemberID="viewController">
<view key="view" contentMode="scaleToFill" id="Ze5-6b-2t3">
<rect key="frame" x="0.0" y="0.0" width="375" height="667"/>
<autoresizingMask key="autoresizingMask" widthSizable="YES" heightSizable="YES"/>
<color key="backgroundColor" red="1" green="1" blue="1" alpha="1" colorSpace="custom" customColorSpace="sRGB"/>
<viewLayoutGuide key="safeArea" id="6Tk-OE-BBY"/>
</view>
</viewController>
<placeholder placeholderIdentifier="IBFirstResponder" id="iYj-Kq-Ea1" userLabel="First Responder" sceneMemberID="firstResponder"/>
</objects>
<point key="canvasLocation" x="53" y="375"/>
</scene>
</scenes>
</document>
mediapipe/examples/ios/faceeffect/Base.lproj/Main.storyboard
+57
View File
@@ -0,0 +1,57 @@
<?xml version="1.0" encoding="UTF-8"?>
<document type="com.apple.InterfaceBuilder3.CocoaTouch.Storyboard.XIB" version="3.0" toolsVersion="16096" targetRuntime="iOS.CocoaTouch" propertyAccessControl="none" useAutolayout="YES" useTraitCollections="YES" useSafeAreas="YES" colorMatched="YES" initialViewController="BYZ-38-t0r">
<device id="retina5_5" orientation="portrait" appearance="light"/>
<dependencies>
<plugIn identifier="com.apple.InterfaceBuilder.IBCocoaTouchPlugin" version="16086"/>
<capability name="Safe area layout guides" minToolsVersion="9.0"/>
<capability name="documents saved in the Xcode 8 format" minToolsVersion="8.0"/>
</dependencies>
<scenes>
<!--View Controller-->
<scene sceneID="tne-QT-ifu">
<objects>
<viewController id="BYZ-38-t0r" customClass="FaceEffectViewController" sceneMemberID="viewController">
<view key="view" contentMode="scaleToFill" id="8bC-Xf-vdC">
<rect key="frame" x="0.0" y="0.0" width="414" height="736"/>
<autoresizingMask key="autoresizingMask" widthSizable="YES" heightSizable="YES"/>
<subviews>
<view contentMode="scaleToFill" fixedFrame="YES" translatesAutoresizingMaskIntoConstraints="NO" id="EfB-xq-knP">
<rect key="frame" x="0.0" y="20" width="414" height="716"/>
<autoresizingMask key="autoresizingMask" widthSizable="YES" heightSizable="YES"/>
<subviews>
<label opaque="NO" userInteractionEnabled="NO" contentMode="left" horizontalHuggingPriority="251" verticalHuggingPriority="251" fixedFrame="YES" text="Camera access needed for this demo. Please enable camera access in the Settings app." textAlignment="center" lineBreakMode="tailTruncation" numberOfLines="0" baselineAdjustment="alignBaselines" adjustsFontSizeToFit="NO" translatesAutoresizingMaskIntoConstraints="NO" id="emf-N5-sEd">
<rect key="frame" x="76" y="283" width="260" height="151"/>
<autoresizingMask key="autoresizingMask" flexibleMinX="YES" flexibleMaxX="YES" flexibleMinY="YES" flexibleMaxY="YES"/>
<fontDescription key="fontDescription" type="system" pointSize="17"/>
<color key="textColor" white="1" alpha="1" colorSpace="custom" customColorSpace="genericGamma22GrayColorSpace"/>
<nil key="highlightedColor"/>
</label>
<label opaque="NO" userInteractionEnabled="NO" contentMode="left" horizontalHuggingPriority="251" verticalHuggingPriority="251" fixedFrame="YES" text="Tap to switch between effects!" textAlignment="center" lineBreakMode="tailTruncation" numberOfLines="0" baselineAdjustment="alignBaselines" adjustsFontSizeToFit="NO" translatesAutoresizingMaskIntoConstraints="NO" id="b2C-zd-Zba" userLabel="Effect Switching Label">
<rect key="frame" x="77" y="488" width="260" height="151"/>
<autoresizingMask key="autoresizingMask" flexibleMinX="YES" flexibleMaxX="YES" flexibleMinY="YES" flexibleMaxY="YES"/>
<fontDescription key="fontDescription" type="system" pointSize="17"/>
<color key="textColor" white="1" alpha="1" colorSpace="custom" customColorSpace="genericGamma22GrayColorSpace"/>
<nil key="highlightedColor"/>
</label>
</subviews>
<color key="backgroundColor" white="0.0" alpha="1" colorSpace="custom" customColorSpace="genericGamma22GrayColorSpace"/>
<accessibility key="accessibilityConfiguration" label="PreviewDisplayView">
<bool key="isElement" value="YES"/>
</accessibility>
</view>
</subviews>
<color key="backgroundColor" red="1" green="1" blue="1" alpha="1" colorSpace="custom" customColorSpace="sRGB"/>
<viewLayoutGuide key="safeArea" id="6Tk-OE-BBY"/>
</view>
<connections>
<outlet property="_effectSwitchingHintLabel" destination="b2C-zd-Zba" id="Uvx-2n-l74"/>
<outlet property="_liveView" destination="EfB-xq-knP" id="JQp-2n-q9q"/>
<outlet property="_noCameraLabel" destination="emf-N5-sEd" id="91G-3Z-cU3"/>
</connections>
</viewController>
<placeholder placeholderIdentifier="IBFirstResponder" id="dkx-z0-nzr" sceneMemberID="firstResponder"/>
</objects>
<point key="canvasLocation" x="48.799999999999997" y="20.239880059970016"/>
</scene>
</scenes>
</document>
mediapipe/examples/ios/faceeffect/FaceEffectViewController.h
+19
View File
@@ -0,0 +1,19 @@
// Copyright 2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#import <UIKit/UIKit.h>
@interface FaceEffectViewController : UIViewController
@end
mediapipe/examples/ios/faceeffect/FaceEffectViewController.mm
+254
View File
@@ -0,0 +1,254 @@
// Copyright 2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#import "FaceEffectViewController.h"
#import "mediapipe/objc/MPPCameraInputSource.h"
#import "mediapipe/objc/MPPGraph.h"
#import "mediapipe/objc/MPPLayerRenderer.h"
#include <utility>
#include "mediapipe/framework/formats/matrix_data.pb.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/modules/face_geometry/protos/face_geometry.pb.h"
static NSString* const kGraphName = @"face_effect_gpu";
static const char* kInputStream = "input_video";
static const char* kIsFacepaintEffectSelectedInputStream = "is_facepaint_effect_selected";
static const char* kOutputStream = "output_video";
static const char* kMultiFaceGeometryStream = "multi_face_geometry";
static const char* kVideoQueueLabel = "com.google.mediapipe.example.videoQueue";
static const int kMatrixTranslationZIndex = 14;
@interface FaceEffectViewController () <MPPGraphDelegate, MPPInputSourceDelegate>
// The MediaPipe graph currently in use. Initialized in viewDidLoad, started in viewWillAppear: and
// sent video frames on _videoQueue.
@property(nonatomic) MPPGraph* graph;
@end
@implementation FaceEffectViewController {
/// Handle tap gestures.
UITapGestureRecognizer* _tapGestureRecognizer;
BOOL _isFacepaintEffectSelected;
/// Handles camera access via AVCaptureSession library.
MPPCameraInputSource* _cameraSource;
/// Inform the user when camera is unavailable.
IBOutlet UILabel* _noCameraLabel;
/// Inform the user about how to switch between effects.
UILabel* _effectSwitchingHintLabel;
/// Display the camera preview frames.
IBOutlet UIView* _liveView;
/// Render frames in a layer.
MPPLayerRenderer* _renderer;
/// Process camera frames on this queue.
dispatch_queue_t _videoQueue;
}
#pragma mark - Cleanup methods
- (void)dealloc {
self.graph.delegate = nil;
[self.graph cancel];
// Ignore errors since we're cleaning up.
[self.graph closeAllInputStreamsWithError:nil];
[self.graph waitUntilDoneWithError:nil];
}
#pragma mark - MediaPipe graph methods
+ (MPPGraph*)loadGraphFromResource:(NSString*)resource {
// Load the graph config resource.
NSError* configLoadError = nil;
NSBundle* bundle = [NSBundle bundleForClass:[self class]];
if (!resource || resource.length == 0) {
return nil;
}
NSURL* graphURL = [bundle URLForResource:resource withExtension:@"binarypb"];
NSData* data = [NSData dataWithContentsOfURL:graphURL options:0 error:&configLoadError];
if (!data) {
NSLog(@"Failed to load MediaPipe graph config: %@", configLoadError);
return nil;
}
// Parse the graph config resource into mediapipe::CalculatorGraphConfig proto object.
mediapipe::CalculatorGraphConfig config;
config.ParseFromArray(data.bytes, data.length);
// Create MediaPipe graph with mediapipe::CalculatorGraphConfig proto object.
MPPGraph* newGraph = [[MPPGraph alloc] initWithGraphConfig:config];
[newGraph addFrameOutputStream:kOutputStream outputPacketType:MPPPacketTypePixelBuffer];
[newGraph addFrameOutputStream:kMultiFaceGeometryStream outputPacketType:MPPPacketTypeRaw];
return newGraph;
}
#pragma mark - UIViewController methods
- (void)viewDidLoad {
[super viewDidLoad];
_effectSwitchingHintLabel.hidden = YES;
_tapGestureRecognizer = [[UITapGestureRecognizer alloc] initWithTarget:self
action:@selector(handleTap)];
[self.view addGestureRecognizer:_tapGestureRecognizer];
// By default, render the glasses effect.
_isFacepaintEffectSelected = NO;
_renderer = [[MPPLayerRenderer alloc] init];
_renderer.layer.frame = _liveView.layer.bounds;
[_liveView.layer insertSublayer:_renderer.layer atIndex:0];
_renderer.frameScaleMode = MPPFrameScaleModeFillAndCrop;
_renderer.mirrored = NO;
dispatch_queue_attr_t qosAttribute = dispatch_queue_attr_make_with_qos_class(
DISPATCH_QUEUE_SERIAL, QOS_CLASS_USER_INTERACTIVE, /*relative_priority=*/0);
_videoQueue = dispatch_queue_create(kVideoQueueLabel, qosAttribute);
_cameraSource = [[MPPCameraInputSource alloc] init];
[_cameraSource setDelegate:self queue:_videoQueue];
_cameraSource.sessionPreset = AVCaptureSessionPresetHigh;
_cameraSource.cameraPosition = AVCaptureDevicePositionFront;
// The frame's native format is rotated with respect to the portrait orientation.
_cameraSource.orientation = AVCaptureVideoOrientationPortrait;
_cameraSource.videoMirrored = YES;
self.graph = [[self class] loadGraphFromResource:kGraphName];
self.graph.delegate = self;
// Set maxFramesInFlight to a small value to avoid memory contention for real-time processing.
self.graph.maxFramesInFlight = 2;
}
// In this application, there is only one ViewController which has no navigation to other view
// controllers, and there is only one View with live display showing the result of running the
// MediaPipe graph on the live video feed. If more view controllers are needed later, the graph
// setup/teardown and camera start/stop logic should be updated appropriately in response to the
// appearance/disappearance of this ViewController, as viewWillAppear: can be invoked multiple times
// depending on the application navigation flow in that case.
- (void)viewWillAppear:(BOOL)animated {
[super viewWillAppear:animated];
[_cameraSource requestCameraAccessWithCompletionHandler:^void(BOOL granted) {
if (granted) {
[self startGraphAndCamera];
dispatch_async(dispatch_get_main_queue(), ^{
_noCameraLabel.hidden = YES;
});
}
}];
}
- (void)startGraphAndCamera {
// Start running self.graph.
NSError* error;
if (![self.graph startWithError:&error]) {
NSLog(@"Failed to start graph: %@", error);
}
// Start fetching frames from the camera.
dispatch_async(_videoQueue, ^{
[_cameraSource start];
});
}
#pragma mark - UITapGestureRecognizer methods
// We use the tap gesture recognizer to switch between face effects. This allows users to try
// multiple pre-bundled face effects without a need to recompile the app.
- (void)handleTap {
dispatch_async(_videoQueue, ^{
_isFacepaintEffectSelected = !_isFacepaintEffectSelected;
});
}
#pragma mark - MPPGraphDelegate methods
// Receives CVPixelBufferRef from the MediaPipe graph. Invoked on a MediaPipe worker thread.
- (void)mediapipeGraph:(MPPGraph*)graph
didOutputPixelBuffer:(CVPixelBufferRef)pixelBuffer
fromStream:(const std::string&)streamName {
if (streamName == kOutputStream) {
// Display the captured image on the screen.
CVPixelBufferRetain(pixelBuffer);
dispatch_async(dispatch_get_main_queue(), ^{
_effectSwitchingHintLabel.hidden = NO;
[_renderer renderPixelBuffer:pixelBuffer];
CVPixelBufferRelease(pixelBuffer);
});
}
}
// Receives a raw packet from the MediaPipe graph. Invoked on a MediaPipe worker thread.
//
// This callback demonstrates how the output face geometry packet can be obtained and used in an
// iOS app. As an example, the Z-translation component of the face pose transform matrix is logged
// for each face being equal to the approximate distance away from the camera in centimeters.
- (void)mediapipeGraph:(MPPGraph*)graph
didOutputPacket:(const ::mediapipe::Packet&)packet
fromStream:(const std::string&)streamName {
if (streamName == kMultiFaceGeometryStream) {
if (packet.IsEmpty()) {
NSLog(@"[TS:%lld] No face geometry", packet.Timestamp().Value());
return;
}
const auto& multiFaceGeometry =
packet.Get<std::vector<::mediapipe::face_geometry::FaceGeometry>>();
NSLog(@"[TS:%lld] Number of face instances with geometry: %lu ", packet.Timestamp().Value(),
multiFaceGeometry.size());
for (int faceIndex = 0; faceIndex < multiFaceGeometry.size(); ++faceIndex) {
const auto& faceGeometry = multiFaceGeometry[faceIndex];
NSLog(@"\tApprox. distance away from camera for face[%d]: %.6f cm", faceIndex,
-faceGeometry.pose_transform_matrix().packed_data(kMatrixTranslationZIndex));
}
}
}
#pragma mark - MPPInputSourceDelegate methods
// Must be invoked on _videoQueue.
- (void)processVideoFrame:(CVPixelBufferRef)imageBuffer
timestamp:(CMTime)timestamp
fromSource:(MPPInputSource*)source {
if (source != _cameraSource) {
NSLog(@"Unknown source: %@", source);
return;
}
mediapipe::Timestamp graphTimestamp(static_cast<mediapipe::TimestampBaseType>(
mediapipe::Timestamp::kTimestampUnitsPerSecond * CMTimeGetSeconds(timestamp)));
mediapipe::Packet isFacepaintEffectSelectedPacket =
mediapipe::MakePacket<bool>(_isFacepaintEffectSelected).At(graphTimestamp);
[self.graph sendPixelBuffer:imageBuffer
intoStream:kInputStream
packetType:MPPPacketTypePixelBuffer
timestamp:graphTimestamp];
// Alongside the input camera frame, we also send the `is_facepaint_effect_selected` boolean
// packet to indicate which effect should be rendered on this frame.
[self.graph movePacket:std::move(isFacepaintEffectSelectedPacket)
intoStream:kIsFacepaintEffectSelectedInputStream
error:nil];
}
@end
mediapipe/examples/ios/faceeffect/Info.plist
+42
View File
@@ -0,0 +1,42 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>NSCameraUsageDescription</key>
<string>This app uses the camera to demonstrate live video processing.</string>
<key>CFBundleDevelopmentRegion</key>
<string>en</string>
<key>CFBundleExecutable</key>
<string>$(EXECUTABLE_NAME)</string>
<key>CFBundleIdentifier</key>
<string>$(PRODUCT_BUNDLE_IDENTIFIER)</string>
<key>CFBundleInfoDictionaryVersion</key>
<string>6.0</string>
<key>CFBundleName</key>
<string>$(PRODUCT_NAME)</string>
<key>CFBundlePackageType</key>
<string>APPL</string>
<key>CFBundleShortVersionString</key>
<string>1.0</string>
<key>CFBundleVersion</key>
<string>1</string>
<key>LSRequiresIPhoneOS</key>
<true/>
<key>UILaunchStoryboardName</key>
<string>LaunchScreen</string>
<key>UIMainStoryboardFile</key>
<string>Main</string>
<key>UIRequiredDeviceCapabilities</key>
<array>
<string>armv7</string>
</array>
<key>UISupportedInterfaceOrientations</key>
<array>
<string>UIInterfaceOrientationPortrait</string>
</array>
<key>UISupportedInterfaceOrientations~ipad</key>
<array>
<string>UIInterfaceOrientationPortrait</string>
</array>
</dict>
</plist>
mediapipe/examples/ios/faceeffect/main.m
+22
View File
@@ -0,0 +1,22 @@
// Copyright 2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#import <UIKit/UIKit.h>
#import "AppDelegate.h"
int main(int argc, char* argv[]) {
@autoreleasepool {
return UIApplicationMain(argc, argv, nil, NSStringFromClass([AppDelegate class]));
}
}
mediapipe/examples/python/upper_body_pose_tracker.py
+1 -1
View File
@@ -150,7 +150,7 @@ class UpperBodyPoseTracker:
success, input_frame = cap.read()
if not success:
break
input_frame = cv2.cvtColor(input_frame, cv2.COLOR_BGR2RGB)
input_frame = cv2.cvtColor(cv2.flip(input_frame, 1), cv2.COLOR_BGR2RGB)
input_frame.flags.writeable = False
_, output_frame = self._run_graph(input_frame)
cv2.imshow('MediaPipe upper body pose tracker',
mediapipe/framework/BUILD
+1
View File
@@ -947,6 +947,7 @@ cc_library(
],
}),
visibility = [
"//mediapipe/calculators:__subpackages__",
"//mediapipe/framework:__subpackages__",
"//mediapipe/framework/port:__pkg__",
"//mediapipe/util:__subpackages__",
mediapipe/framework/calculator_graph.cc
+3
View File
@@ -330,6 +330,9 @@ CalculatorGraph::~CalculatorGraph() {
::mediapipe::Status CalculatorGraph::InitializeDefaultExecutor(
const ThreadPoolExecutorOptions* default_executor_options,
bool use_application_thread) {
#ifdef __EMSCRIPTEN__
use_application_thread = true;
#endif // __EMSCRIPTEN__
// If specified, run synchronously on the calling thread.
if (use_application_thread) {
use_application_thread_ = true;
mediapipe/framework/deps/file_helpers.cc
+28 -1
View File
@@ -27,7 +27,9 @@
#include "mediapipe/framework/deps/canonical_errors.h"
#include "mediapipe/framework/deps/file_path.h"
#include "mediapipe/framework/deps/status.h"
#include "mediapipe/framework/deps/status_builder.h"
#include "mediapipe/framework/deps/status_macros.h"
namespace mediapipe {
namespace file {
@@ -212,7 +214,7 @@ class DirectoryListing {
::mediapipe::Status Exists(absl::string_view file_name) {
struct stat buffer;
int status;
status = stat(file_name.data(), &buffer);
status = stat(std::string(file_name).c_str(), &buffer);
if (status == 0) {
return ::mediapipe::OkStatus();
}
@@ -224,5 +226,30 @@ class DirectoryListing {
}
}
#ifndef _WIN32
int mkdir(std::string path) {
return ::mkdir(path.c_str(), S_IRWXU | S_IRWXG | S_IRWXO);
}
#else
int mkdir(std::string path) { return _mkdir(path.c_str()); }
#endif
::mediapipe::Status RecursivelyCreateDir(absl::string_view path) {
if (path.empty() || Exists(path).ok()) {
return mediapipe::OkStatus();
}
auto split_path = file::SplitPath(path);
MP_RETURN_IF_ERROR(RecursivelyCreateDir(split_path.first));
if (mkdir(std::string(path)) != 0) {
switch (errno) {
case EACCES:
return ::mediapipe::PermissionDeniedError("Insufficient permissions.");
default:
return ::mediapipe::UnavailableError("Failed to create directory.");
}
}
return mediapipe::OkStatus();
}
} // namespace file
} // namespace mediapipe
mediapipe/framework/deps/file_helpers.h
+2
View File
@@ -36,6 +36,8 @@ namespace file {
::mediapipe::Status Exists(absl::string_view file_name);
::mediapipe::Status RecursivelyCreateDir(absl::string_view path);
} // namespace file
} // namespace mediapipe
mediapipe/framework/deps/threadpool_pthread_impl.cc
+13 -3
View File
@@ -47,7 +47,8 @@ class ThreadPool::WorkerThread {
ThreadPool::WorkerThread::WorkerThread(ThreadPool* pool,
const std::string& name_prefix)
: pool_(pool), name_prefix_(name_prefix) {
pthread_create(&thread_, nullptr, ThreadBody, this);
int res = pthread_create(&thread_, nullptr, ThreadBody, this);
CHECK_EQ(res, 0) << "pthread_create failed";
}
ThreadPool::WorkerThread::~WorkerThread() {}
@@ -59,9 +60,9 @@ void* ThreadPool::WorkerThread::ThreadBody(void* arg) {
int nice_priority_level =
thread->pool_->thread_options().nice_priority_level();
const std::set<int> selected_cpus = thread->pool_->thread_options().cpu_set();
#if defined(__linux__)
const std::string name =
internal::CreateThreadName(thread->name_prefix_, syscall(SYS_gettid));
#if defined(__linux__)
if (nice_priority_level != 0) {
if (nice(nice_priority_level) != -1 || errno == 0) {
VLOG(1) << "Changed the nice priority level by " << nice_priority_level;
@@ -94,16 +95,19 @@ void* ThreadPool::WorkerThread::ThreadBody(void* arg) {
<< "Failed to set name for thread: " << name;
}
#else
const std::string name = internal::CreateThreadName(thread->name_prefix_, 0);
if (nice_priority_level != 0 || !selected_cpus.empty()) {
LOG(ERROR) << "Thread priority and processor affinity feature aren't "
"supported on the current platform.";
}
#if __APPLE__
int error = pthread_setname_np(name.c_str());
if (error != 0) {
LOG(ERROR) << "Error : " << strerror(error) << std::endl
<< "Failed to set name for thread: " << name;
}
#endif
#endif // __APPLE__
#endif // __linux__
thread->pool_->RunWorker();
return nullptr;
}
@@ -178,6 +182,12 @@ const ThreadOptions& ThreadPool::thread_options() const {
namespace internal {
// TODO: revise this:
// - thread_id is not portable
// - the 16-byte limit is Linux-specific
// - the std::thread implementation has a copy of this but doesn't use it
// - why do we even need the thread id in the name? any thread list should show
// the id too.
std::string CreateThreadName(const std::string& prefix, int thread_id) {
std::string name = absl::StrCat(prefix, "/", thread_id);
// 16 is the limit allowed by `pthread_setname_np`, including
mediapipe/framework/encode_binary_proto.bzl
+14
View File
@@ -1,3 +1,17 @@
# Copyright 2019-2020 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""A rule for encoding a text format protocol buffer into binary.
Example usage:
mediapipe/framework/formats/BUILD
+25
View File
@@ -227,3 +227,28 @@ filegroup(
srcs = glob(["*.proto"]),
visibility = ["//mediapipe:__subpackages__"],
)
cc_library(
name = "image_frame_pool",
srcs = ["image_frame_pool.cc"],
hdrs = ["image_frame_pool.h"],
visibility = ["//visibility:public"],
deps = [
":image_frame",
"@com_google_absl//absl/memory",
"@com_google_absl//absl/synchronization",
],
)
cc_test(
name = "image_frame_pool_test",
size = "small",
srcs = ["image_frame_pool_test.cc"],
tags = ["linux"],
deps = [
":image_frame_pool",
"//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:status",
"@com_google_absl//absl/memory",
],
)
mediapipe/framework/formats/image_frame_pool.cc
+88
View File
@@ -0,0 +1,88 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/framework/formats/image_frame_pool.h"
#include "absl/synchronization/mutex.h"
namespace mediapipe {
ImageFramePool::ImageFramePool(int width, int height,
ImageFormat::Format format, int keep_count)
: width_(width),
height_(height),
format_(format),
keep_count_(keep_count) {}
ImageFrameSharedPtr ImageFramePool::GetBuffer() {
std::unique_ptr<ImageFrame> buffer;
{
absl::MutexLock lock(&mutex_);
if (available_.empty()) {
// Fix alignment at 4 for best compatability with OpenGL.
buffer = std::make_unique<ImageFrame>(
format_, width_, height_, ImageFrame::kGlDefaultAlignmentBoundary);
if (!buffer) return nullptr;
} else {
buffer = std::move(available_.back());
available_.pop_back();
}
++in_use_count_;
}
// Return a shared_ptr with a custom deleter that adds the buffer back
// to our available list.
std::weak_ptr<ImageFramePool> weak_pool(shared_from_this());
return std::shared_ptr<ImageFrame>(buffer.release(),
[weak_pool](ImageFrame* buf) {
auto pool = weak_pool.lock();
if (pool) {
pool->Return(buf);
} else {
delete buf;
}
});
}
std::pair<int, int> ImageFramePool::GetInUseAndAvailableCounts() {
absl::MutexLock lock(&mutex_);
return {in_use_count_, available_.size()};
}
void ImageFramePool::Return(ImageFrame* buf) {
std::vector<std::unique_ptr<ImageFrame>> trimmed;
{
absl::MutexLock lock(&mutex_);
--in_use_count_;
available_.emplace_back(buf);
TrimAvailable(&trimmed);
}
// The trimmed buffers will be released without holding the lock.
}
void ImageFramePool::TrimAvailable(
std::vector<std::unique_ptr<ImageFrame>>* trimmed) {
int keep = std::max(keep_count_ - in_use_count_, 0);
if (available_.size() > keep) {
auto trim_it = std::next(available_.begin(), keep);
if (trimmed) {
std::move(trim_it, available_.end(), std::back_inserter(*trimmed));
}
available_.erase(trim_it, available_.end());
}
}
} // namespace mediapipe
mediapipe/framework/formats/image_frame_pool.h
+78
View File
@@ -0,0 +1,78 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Consider this file an implementation detail. None of this is part of the
// public API.
#ifndef MEDIAPIPE_FRAMEWORK_FORMATS_IMAGE_FRAME_POOL_H_
#define MEDIAPIPE_FRAMEWORK_FORMATS_IMAGE_FRAME_POOL_H_
#include <utility>
#include <vector>
#include "absl/synchronization/mutex.h"
#include "mediapipe/framework/formats/image_frame.h"
namespace mediapipe {
using ImageFrameSharedPtr = std::shared_ptr<ImageFrame>;
class ImageFramePool : public std::enable_shared_from_this<ImageFramePool> {
public:
// Creates a pool. This pool will manage buffers of the specified dimensions,
// and will keep keep_count buffers around for reuse.
// We enforce creation as a shared_ptr so that we can use a weak reference in
// the buffers' deleters.
static std::shared_ptr<ImageFramePool> Create(int width, int height,
ImageFormat::Format format,
int keep_count) {
return std::shared_ptr<ImageFramePool>(
new ImageFramePool(width, height, format, keep_count));
}
// Obtains a buffers. May either be reused or created anew.
ImageFrameSharedPtr GetBuffer();
int width() const { return width_; }
int height() const { return height_; }
ImageFormat::Format format() const { return format_; }
// This method is meant for testing.
std::pair<int, int> GetInUseAndAvailableCounts();
private:
ImageFramePool(int width, int height, ImageFormat::Format format,
int keep_count);
// Return a buffer to the pool.
void Return(ImageFrame* buf);
// If the total number of buffers is greater than keep_count, destroys any
// surplus buffers that are no longer in use.
void TrimAvailable(std::vector<std::unique_ptr<ImageFrame>>* trimmed)
ABSL_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
const int width_;
const int height_;
const ImageFormat::Format format_;
const int keep_count_;
absl::Mutex mutex_;
int in_use_count_ ABSL_GUARDED_BY(mutex_) = 0;
std::vector<std::unique_ptr<ImageFrame>> available_ ABSL_GUARDED_BY(mutex_);
};
} // namespace mediapipe
#endif // MEDIAPIPE_FRAMEWORK_FORMATS_IMAGE_FRAME_POOL_H_
mediapipe/framework/formats/image_frame_pool_test.cc
+103
View File
@@ -0,0 +1,103 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/framework/formats/image_frame_pool.h"
#include "absl/memory/memory.h"
#include "mediapipe/framework/port/gmock.h"
#include "mediapipe/framework/port/gtest.h"
#include "mediapipe/framework/port/status.h"
namespace mediapipe {
namespace {
using Pair = std::pair<int, int>;
constexpr int kWidth = 300;
constexpr int kHeight = 200;
constexpr ImageFormat::Format kFormat = ImageFormat::SRGBA;
constexpr int kKeepCount = 2;
class ImageFramePoolTest : public ::testing::Test {
protected:
ImageFramePoolTest() {
pool_ = ImageFramePool::Create(kWidth, kHeight, kFormat, kKeepCount);
}
void SetUp() override {}
std::shared_ptr<ImageFramePool> pool_;
};
TEST_F(ImageFramePoolTest, GetBuffer) {
EXPECT_EQ(Pair(0, 0), pool_->GetInUseAndAvailableCounts());
auto buffer = pool_->GetBuffer();
EXPECT_EQ(Pair(1, 0), pool_->GetInUseAndAvailableCounts());
buffer = nullptr;
EXPECT_EQ(Pair(0, 1), pool_->GetInUseAndAvailableCounts());
buffer = pool_->GetBuffer();
EXPECT_EQ(Pair(1, 0), pool_->GetInUseAndAvailableCounts());
}
TEST_F(ImageFramePoolTest, GetMoreBuffers) {
EXPECT_EQ(Pair(0, 0), pool_->GetInUseAndAvailableCounts());
std::vector<ImageFrameSharedPtr> buffers;
// Create kKeepCount + 1 buffers
for (int i = 0; i <= kKeepCount; i++) {
buffers.emplace_back(pool_->GetBuffer());
}
EXPECT_EQ(Pair(kKeepCount + 1, 0), pool_->GetInUseAndAvailableCounts());
// Delete one
buffers.resize(kKeepCount);
EXPECT_EQ(Pair(kKeepCount, 0), pool_->GetInUseAndAvailableCounts());
// Delete all
buffers.resize(0);
EXPECT_EQ(Pair(0, kKeepCount), pool_->GetInUseAndAvailableCounts());
// Create one more
buffers.emplace_back(pool_->GetBuffer());
EXPECT_EQ(Pair(1, kKeepCount - 1), pool_->GetInUseAndAvailableCounts());
}
TEST_F(ImageFramePoolTest, DeleteNotLast) {
EXPECT_EQ(Pair(0, 0), pool_->GetInUseAndAvailableCounts());
std::vector<ImageFrameSharedPtr> buffers;
// Create kKeepCount + 1 buffers
for (int i = 0; i <= kKeepCount; i++) {
buffers.emplace_back(pool_->GetBuffer());
}
EXPECT_EQ(Pair(kKeepCount + 1, 0), pool_->GetInUseAndAvailableCounts());
// Delete second
buffers.erase(buffers.begin() + 1);
EXPECT_EQ(Pair(kKeepCount, 0), pool_->GetInUseAndAvailableCounts());
// Delete first
buffers.erase(buffers.begin());
EXPECT_EQ(Pair(kKeepCount - 1, 1), pool_->GetInUseAndAvailableCounts());
}
TEST(ImageFrameBufferPoolStaticTest, BufferCanOutlivePool) {
auto pool = ImageFramePool::Create(kWidth, kHeight, kFormat, kKeepCount);
auto buffer = pool->GetBuffer();
pool = nullptr;
buffer = nullptr;
}
} // anonymous namespace
} // namespace mediapipe
mediapipe/framework/mediapipe_register_type.bzl
+117
View File
@@ -0,0 +1,117 @@
# Copyright 2020 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""A rule for registering types with mediapipe.
Example usage:
mediapipe_proto_library(
name = "foo_proto",
srcs = ["foo.proto"],
)
load("//mediapipe/framework:mediapipe_register_type.bzl",
"mediapipe_register_type")
# Creates rules "foo_registration"
mediapipe_register_type(
base_name = "foo"
include_headers = ["mediapipe/framework/formats/foo.proto.h""],
types = [
"::mediapipe::Foo",
"::mediapipe::FooList",
"::std::vector<::mediapipe::Foo>",
],
deps = [":foo_cc_proto"],
)
Args
base_name: The base name of the target
(name + "_registration" will be created).
types: A list of C++ classes to register using MEDIAPIPE_REGISTER_TYPE.
deps: A list of cc deps.
include_headers: A list of header files that must be included.
"""
load("//mediapipe/framework/tool:build_defs.bzl", "clean_dep")
def _mediapipe_register_type_generate_cc_impl(ctx):
"""Generate a cc file that registers types with mediapipe."""
file_data_template = '''
{include_headers}
#include "mediapipe/framework/type_map.h"
{registration_commands}
'''
header_lines = []
for header in ctx.attr.include_headers:
header_lines.append('#include "{}"'.format(header))
registration_lines = []
for registration_type in ctx.attr.types:
if " " in registration_type:
fail(('registration type "{}" should be fully qualified ' +
"and must not include spaces").format(registration_type))
registration_lines.append(
"#define TEMP_MP_TYPE {}".format(registration_type),
)
registration_lines.append(
("MEDIAPIPE_REGISTER_TYPE(\n" +
" TEMP_MP_TYPE,\n" +
' "{}",\n'.format(registration_type) +
" nullptr, nullptr);\n"),
)
registration_lines.append("#undef TEMP_MP_TYPE")
file_data = file_data_template.format(
include_headers = "\n".join(header_lines),
registration_commands = "\n".join(registration_lines),
)
ctx.actions.write(ctx.outputs.output, file_data)
mediapipe_register_type_generate_cc = rule(
implementation = _mediapipe_register_type_generate_cc_impl,
attrs = {
"deps": attr.label_list(),
"types": attr.string_list(
mandatory = True,
),
"include_headers": attr.string_list(
mandatory = True,
),
"output": attr.output(),
},
)
def mediapipe_register_type(
base_name,
types,
deps = [],
include_headers = [],
visibility = ["//visibility:public"]):
mediapipe_register_type_generate_cc(
name = base_name + "_registration_cc",
types = types,
include_headers = include_headers,
output = base_name + "_registration.cc",
)
native.cc_library(
name = base_name + "_registration",
srcs = [base_name + "_registration.cc"],
deps = depset(deps + [
clean_dep("//mediapipe/framework:type_map"),
]),
visibility = visibility,
alwayslink = 1,
)
mediapipe/framework/port.h
+5 -5
View File
@@ -52,12 +52,12 @@
// Compile time target platform definitions.
// Example: #if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31
#define MEDIAPIPE_OPENGL_ES_UNSUPPORTED 0
#define MEDIAPIPE_OPENGL_ES_20 200
#define MEDIAPIPE_OPENGL_ES_30 300
#define MEDIAPIPE_OPENGL_ES_31 310
#if defined(MEDIAPIPE_DISABLE_GPU)
#define MEDIAPIPE_OPENGL_ES_VERSION MEDIAPIPE_OPENGL_ES_UNSUPPORTED
#define MEDIAPIPE_OPENGL_ES_VERSION 0
#define MEDIAPIPE_METAL_ENABLED 0
#else
#if defined(MEDIAPIPE_ANDROID)
@@ -71,11 +71,11 @@
#define MEDIAPIPE_OPENGL_ES_VERSION MEDIAPIPE_OPENGL_ES_20
#define MEDIAPIPE_METAL_ENABLED 1
#elif defined(MEDIAPIPE_OSX)
#define MEDIAPIPE_OPENGL_ES_VERSION MEDIAPIPE_OPENGL_ES_UNSUPPORTED
#define MEDIAPIPE_OPENGL_ES_VERSION 0
#define MEDIAPIPE_METAL_ENABLED 1
#else
// GPU is not supported on Linux yet.
#define MEDIAPIPE_OPENGL_ES_VERSION MEDIAPIPE_OPENGL_ES_UNSUPPORTED
// WebGL config.
#define MEDIAPIPE_OPENGL_ES_VERSION MEDIAPIPE_OPENGL_ES_30
#define MEDIAPIPE_METAL_ENABLED 0
#endif
#endif
mediapipe/framework/profiler/BUILD
+9 -2
View File
@@ -269,7 +269,7 @@ cc_test(
cc_library(
name = "profiler_resource_util",
srcs = select({
srcs = ["profiler_resource_util_common.cc"] + select({
"//conditions:default": ["profiler_resource_util.cc"],
"//mediapipe:android": ["profiler_resource_util_android.cc"],
"//mediapipe:ios": ["profiler_resource_util_ios.cc"],
@@ -288,15 +288,22 @@ cc_library(
deps = [
"@com_google_absl//absl/strings",
"//mediapipe/framework/port:logging",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:statusor",
"//mediapipe/framework/port:status",
"@com_google_absl//absl/flags:flag",
"//mediapipe/framework/deps:file_path",
] + select({
"//conditions:default": [
"//mediapipe/framework/port:file_helpers",
],
"//mediapipe:android": [
"//mediapipe/java/com/google/mediapipe/framework/jni:jni_util",
"//mediapipe/util/android/file/base",
],
"//mediapipe:apple": [
"//mediapipe/framework/port:file_helpers",
],
"//mediapipe:apple": [],
}),
)
mediapipe/framework/profiler/graph_profiler_test.cc
+1
View File
@@ -22,6 +22,7 @@
#include "mediapipe/framework/port/gmock.h"
#include "mediapipe/framework/port/gtest.h"
#include "mediapipe/framework/port/status_matchers.h"
#include "mediapipe/framework/port/statusor.h"
#include "mediapipe/framework/profiler/test_context_builder.h"
#include "mediapipe/framework/tool/simulation_clock.h"
#include "mediapipe/framework/tool/tag_map_helper.h"
mediapipe/framework/profiler/profiler_resource_util.h
+4
View File
@@ -27,6 +27,10 @@ namespace mediapipe {
// error.
StatusOr<std::string> GetDefaultTraceLogDirectory();
// Given a log file path, this function provides an absolute path with which
// it can be accessed as a file. Enclosing directories are created as needed.
StatusOr<std::string> PathToLogFile(const std::string& path);
} // namespace mediapipe
#endif // MEDIAPIPE_FRAMEWORK_PROFILER_PROFILER_RESOURCE_UTIL_H_
mediapipe/framework/profiler/profiler_resource_util_common.cc
+60
View File
@@ -0,0 +1,60 @@
// Copyright 2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "absl/flags/flag.h"
#include "mediapipe/framework/deps/file_path.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status_macros.h"
#include "mediapipe/framework/profiler/profiler_resource_util.h"
// TODO: Move this Android include to port/file_helpers.
// Also move this from resource_util.cc.
#ifdef __ANDROID__
#include "mediapipe/util/android/file/base/filesystem.h"
#else
#include "mediapipe/framework/port/file_helpers.h"
#endif
ABSL_FLAG(std::string, log_root_dir, "",
"The absolute path to the logging output directory. If specified, "
"log_root_dir will be prepended to each specified log file path.");
#ifdef __ANDROID__
namespace mediapipe {
namespace file {
::mediapipe::Status RecursivelyCreateDir(absl::string_view path) {
return RecursivelyCreateDir(path, file::Options());
}
} // namespace file
} // namespace mediapipe
#endif
namespace mediapipe {
::mediapipe::StatusOr<std::string> GetLogDirectory() {
if (!FLAGS_log_root_dir.CurrentValue().empty()) {
return FLAGS_log_root_dir.CurrentValue();
}
return GetDefaultTraceLogDirectory();
}
::mediapipe::StatusOr<std::string> PathToLogFile(const std::string& path) {
ASSIGN_OR_RETURN(std::string log_dir, GetLogDirectory());
std::string result = file::JoinPath(log_dir, path);
MP_RETURN_IF_ERROR(
::mediapipe::file::RecursivelyCreateDir(file::Dirname(result)));
return result;
}
} // namespace mediapipe
mediapipe/framework/stream_handler/mux_input_stream_handler.cc
+11
View File
@@ -14,6 +14,7 @@
#include "absl/strings/substitute.h"
#include "absl/synchronization/mutex.h"
#include "mediapipe/framework/collection_item_id.h"
#include "mediapipe/framework/input_stream_handler.h"
#include "mediapipe/framework/port/logging.h"
@@ -121,6 +122,16 @@ class MuxInputStreamHandler : public InputStreamHandler {
num_packets_dropped, data_stream->Name());
AddPacketToShard(&input_set->Get(data_stream_id), std::move(data_packet),
stream_is_done);
// Discard old packets on other streams.
// Note that control_stream_id is the last valid id.
auto next_timestamp = input_timestamp.NextAllowedInStream();
for (CollectionItemId id = input_stream_managers_.BeginId();
id < control_stream_id; ++id) {
if (id == data_stream_id) continue;
auto& other_stream = input_stream_managers_.Get(id);
other_stream->ErasePacketsEarlierThan(next_timestamp);
}
}
private:
mediapipe/framework/tool/simulation_clock_executor.cc
+2 -2
View File
@@ -19,11 +19,11 @@
namespace mediapipe {
SimulationClockExecutor::SimulationClockExecutor(int num_threads)
: ThreadPoolExecutor(num_threads), clock_(new SimulationClock()) {}
: clock_(new SimulationClock()), executor_(num_threads) {}
void SimulationClockExecutor::Schedule(std::function<void()> task) {
clock_->ThreadStart();
ThreadPoolExecutor::Schedule([this, task] {
executor_.Schedule([this, task] {
clock_->Sleep(absl::ZeroDuration());
task();
clock_->ThreadFinish();
mediapipe/framework/tool/simulation_clock_executor.h
+4 -1
View File
@@ -23,7 +23,7 @@ namespace mediapipe {
// Simulation clock multithreaded executor. This is intended to be used with
// graphs that are using SimulationClock class to emulate various parts of the
// graph taking specific time to process the incoming packets.
class SimulationClockExecutor : public ThreadPoolExecutor {
class SimulationClockExecutor : public Executor {
public:
explicit SimulationClockExecutor(int num_threads);
void Schedule(std::function<void()> task) override;
@@ -36,6 +36,9 @@ class SimulationClockExecutor : public ThreadPoolExecutor {
private:
// SimulationClock instance used by this executor.
std::shared_ptr<SimulationClock> clock_;
// The delegate ThreadPoolExecutor. This is declared after clock_
// so that it is destroyed before clock_.
ThreadPoolExecutor executor_;
};
} // namespace mediapipe
mediapipe/framework/transitive_protos.bzl
+14
View File
@@ -1,3 +1,17 @@
# Copyright 2019-2020 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Extract a cc_library compatible dependency with only the top level proto rules."""
ProtoLibsInfo = provider(fields = ["targets", "out"])
mediapipe/gpu/BUILD
+22 -12
View File
@@ -159,7 +159,7 @@ cc_library(
"//conditions:default": [],
"//mediapipe:apple": [
"-x objective-c++",
"-fobjc-arc",
"-fobjc-arc", # enable reference-counting
],
}),
visibility = ["//visibility:public"],
@@ -407,20 +407,29 @@ cc_library(
}),
)
cc_library(
name = "gl_texture_buffer_pool",
srcs = ["gl_texture_buffer_pool.cc"],
hdrs = ["gl_texture_buffer_pool.h"],
visibility = ["//visibility:public"],
deps = [
":gl_base",
":gl_texture_buffer",
":gpu_buffer",
":gpu_shared_data_header",
"//mediapipe/framework:calculator_context",
"//mediapipe/framework:calculator_node",
"//mediapipe/framework/port:logging",
"@com_google_absl//absl/memory",
"@com_google_absl//absl/synchronization",
],
)
cc_library(
name = "gpu_buffer_multi_pool",
srcs = ["gpu_buffer_multi_pool.cc"] + select({
"//conditions:default": [
"gl_texture_buffer_pool.cc",
],
"//mediapipe:ios": [],
"//mediapipe:macos": [
"gl_texture_buffer_pool.cc",
],
}),
srcs = ["gpu_buffer_multi_pool.cc"],
hdrs = ["gpu_buffer_multi_pool.h"] + select({
"//conditions:default": [
"gl_texture_buffer_pool.h",
],
"//mediapipe:ios": [
# The inclusions check does not see that this is provided by
@@ -429,7 +438,6 @@ cc_library(
"pixel_buffer_pool_util.h",
],
"//mediapipe:macos": [
"gl_texture_buffer_pool.h",
],
}),
copts = select({
@@ -452,6 +460,7 @@ cc_library(
] + select({
"//conditions:default": [
":gl_texture_buffer",
":gl_texture_buffer_pool",
],
"//mediapipe:ios": [
":pixel_buffer_pool_util",
@@ -460,6 +469,7 @@ cc_library(
"//mediapipe:macos": [
":pixel_buffer_pool_util",
":gl_texture_buffer",
":gl_texture_buffer_pool",
],
}),
)
mediapipe/gpu/gl_calculator_helper.cc
+6 -1
View File
@@ -14,14 +14,19 @@
#include "mediapipe/gpu/gl_calculator_helper.h"
#include "absl/memory/memory.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/legacy_calculator_support.h"
#include "mediapipe/framework/port/canonical_errors.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/gpu/gl_calculator_helper_impl.h"
#include "mediapipe/gpu/gpu_buffer.h"
#include "mediapipe/gpu/gpu_service.h"
#ifdef __APPLE__
#include "mediapipe/objc/util.h"
#endif
namespace mediapipe {
GlTexture::GlTexture(GLuint name, int width, int height)
mediapipe/gpu/gl_calculator_helper.h
+20 -7
View File
@@ -15,8 +15,6 @@
#ifndef MEDIAPIPE_GPU_GL_CALCULATOR_HELPER_H_
#define MEDIAPIPE_GPU_GL_CALCULATOR_HELPER_H_
#include <memory>
#include "absl/memory/memory.h"
#include "mediapipe/framework/calculator_context.h"
#include "mediapipe/framework/calculator_contract.h"
@@ -29,6 +27,7 @@
#include "mediapipe/gpu/gl_context.h"
#include "mediapipe/gpu/gpu_buffer.h"
#include "mediapipe/gpu/graph_support.h"
#ifdef __APPLE__
#include <CoreVideo/CoreVideo.h>
@@ -50,6 +49,8 @@ typedef CVOpenGLESTextureRef CVTextureType;
#endif // TARGET_OS_OSX
#endif // __APPLE__
using ImageFrameSharedPtr = std::shared_ptr<ImageFrame>;
// TODO: remove this and Process below, or make Process available
// on Android.
typedef std::function<void(const GlTexture& src, const GlTexture& dst)>
@@ -121,7 +122,7 @@ class GlCalculatorHelper {
// where it is supported (iOS, for now) they take advantage of memory sharing
// between the CPU and GPU, avoiding memory copies.
// Creates a texture representing an input frame.
// Creates a texture representing an input frame, and manages sync token.
GlTexture CreateSourceTexture(const GpuBuffer& pixel_buffer);
GlTexture CreateSourceTexture(const ImageFrame& image_frame);
@@ -137,7 +138,7 @@ class GlCalculatorHelper {
void GetGpuBufferDimensions(const GpuBuffer& pixel_buffer, int* width,
int* height);
// Creates a texture representing an output frame.
// Creates a texture representing an output frame, and manages sync token.
// TODO: This should either return errors or a status.
GlTexture CreateDestinationTexture(
int output_width, int output_height,
@@ -152,11 +153,22 @@ class GlCalculatorHelper {
GlContext& GetGlContext() const;
// Check if the calculator helper has been previously initialized.
bool Initialized() { return impl_ != nullptr; }
private:
std::unique_ptr<GlCalculatorHelperImpl> impl_;
};
// Represents an OpenGL texture.
// Represents an OpenGL texture, and is a 'view' into the memory pool.
// It's more like a GlTextureLock, because it's main purpose (in conjunction
// with the helper): to manage GL sync points in the gl command queue.
//
// This class should be the main way to interface with GL memory within a single
// calculator. This is the preferred way to utilize the memory pool inside of
// the helper, because GlTexture manages efficiently releasing memory back into
// the pool. A GPU backed Image can be extracted from the unerlying
// memory.
class GlTexture {
public:
GlTexture() {}
@@ -170,11 +182,12 @@ class GlTexture {
GLuint name() const { return name_; }
// Returns a buffer that can be sent to another calculator.
// Can be used with GpuBuffer or ImageFrame.
// & manages sync token
// Can be used with GpuBuffer or ImageFrame or Image
template <typename T>
std::unique_ptr<T> GetFrame() const;
// Releases texture memory
// Releases texture memory & manages sync token
void Release();
private:
mediapipe/gpu/gl_calculator_helper_impl.h
+2 -2
View File
@@ -42,10 +42,10 @@ class GlCalculatorHelperImpl {
CalculatorContext* calculator_context);
GlTexture CreateSourceTexture(const ImageFrame& image_frame);
GlTexture CreateSourceTexture(const GpuBuffer& pixel_buffer);
GlTexture CreateSourceTexture(const GpuBuffer& gpu_buffer);
// Note: multi-plane support is currently only available on iOS.
GlTexture CreateSourceTexture(const GpuBuffer& pixel_buffer, int plane);
GlTexture CreateSourceTexture(const GpuBuffer& gpu_buffer, int plane);
// Creates a framebuffer and returns the texture that it is bound to.
GlTexture CreateDestinationTexture(int output_width, int output_height,
mediapipe/gpu/gl_context_egl.cc
+21 -12
View File
@@ -177,6 +177,27 @@ GlContext::StatusOrGlContext GlContext::Create(EGLContext share_context,
}
void GlContext::DestroyContext() {
#ifdef __ANDROID__
if (HasContext()) {
// Detach the current program to work around b/166322604.
auto detach_program = [this] {
GlContext::ContextBinding saved_context;
GetCurrentContextBinding(&saved_context);
// Note: cannot use ThisContextBinding because it calls shared_from_this,
// which is not available during destruction.
if (eglMakeCurrent(display_, surface_, surface_, context_)) {
glUseProgram(0);
} else {
LOG(ERROR) << "eglMakeCurrent() returned error " << std::showbase
<< std::hex << eglGetError();
}
return SetCurrentContextBinding(saved_context);
};
auto status = thread_ ? thread_->Run(detach_program) : detach_program();
LOG_IF(ERROR, !status.ok()) << status;
}
#endif // __ANDROID__
if (thread_) {
// Delete thread-local storage.
// TODO: in theory our EglThreadExitCallback should suffice for
@@ -191,18 +212,6 @@ void GlContext::DestroyContext() {
.IgnoreError();
}
#ifdef __ANDROID__
if (HasContext()) {
// Detach the current program to work around b/166322604.
if (eglMakeCurrent(display_, surface_, surface_, context_)) {
glUseProgram(0);
} else {
LOG(ERROR) << "eglMakeCurrent() returned error " << std::showbase
<< std::hex << eglGetError();
}
}
#endif // __ANDROID__
// Destroy the context and surface.
if (IsCurrent()) {
if (!eglMakeCurrent(display_, EGL_NO_SURFACE, EGL_NO_SURFACE,
mediapipe/gpu/gpu_buffer.h
-1
View File
@@ -72,7 +72,6 @@ class GpuBuffer {
int width() const;
int height() const;
GpuBufferFormat format() const;
// Converts to true iff valid.
mediapipe/gpu/gpu_buffer_multi_pool.cc
+3 -3
View File
@@ -44,8 +44,8 @@ static constexpr int kRequestCountScrubInterval = 50;
#if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
CvPixelBufferPoolWrapper::CvPixelBufferPoolWrapper(const BufferSpec& spec,
CFTimeInterval maxAge) {
CvPixelBufferPoolWrapper::CvPixelBufferPoolWrapper(
const GpuBufferMultiPool::BufferSpec& spec, CFTimeInterval maxAge) {
OSType cv_format = CVPixelFormatForGpuBufferFormat(spec.format);
CHECK_NE(cv_format, -1) << "unsupported pixel format";
pool_ = MakeCFHolderAdopting(
@@ -90,7 +90,7 @@ std::string CvPixelBufferPoolWrapper::GetDebugString() const {
void CvPixelBufferPoolWrapper::Flush() { CVPixelBufferPoolFlush(*pool_, 0); }
GpuBufferMultiPool::SimplePool GpuBufferMultiPool::MakeSimplePool(
const BufferSpec& spec) {
const GpuBufferMultiPool::BufferSpec& spec) {
return std::make_shared<CvPixelBufferPoolWrapper>(spec,
kMaxInactiveBufferAge);
}
mediapipe/gpu/gpu_buffer_multi_pool.h
+55 -50
View File
@@ -40,56 +40,7 @@
namespace mediapipe {
struct GpuSharedData;
struct BufferSpec {
BufferSpec(int w, int h, GpuBufferFormat f)
: width(w), height(h), format(f) {}
int width;
int height;
GpuBufferFormat format;
};
inline bool operator==(const BufferSpec& lhs, const BufferSpec& rhs) {
return lhs.width == rhs.width && lhs.height == rhs.height &&
lhs.format == rhs.format;
}
inline bool operator!=(const BufferSpec& lhs, const BufferSpec& rhs) {
return !operator==(lhs, rhs);
}
// This generates a "rol" instruction with both Clang and GCC.
static inline std::size_t RotateLeft(std::size_t x, int n) {
return (x << n) | (x >> (std::numeric_limits<size_t>::digits - n));
}
struct BufferSpecHash {
std::size_t operator()(const mediapipe::BufferSpec& spec) const {
// Width and height are expected to be smaller than half the width of
// size_t. We can combine them into a single integer, and then use
// std::hash, which is what go/hashing recommends for hashing numbers.
constexpr int kWidth = std::numeric_limits<size_t>::digits;
return std::hash<std::size_t>{}(
spec.width ^ RotateLeft(spec.height, kWidth / 2) ^
RotateLeft(static_cast<uint32_t>(spec.format), kWidth / 4));
}
};
#if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
class CvPixelBufferPoolWrapper {
public:
CvPixelBufferPoolWrapper(const BufferSpec& spec, CFTimeInterval maxAge);
GpuBuffer GetBuffer(std::function<void(void)> flush);
int GetBufferCount() const { return count_; }
std::string GetDebugString() const;
void Flush();
private:
CFHolder<CVPixelBufferPoolRef> pool_;
int count_ = 0;
};
#endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
class CvPixelBufferPoolWrapper;
class GpuBufferMultiPool {
public:
@@ -114,6 +65,31 @@ class GpuBufferMultiPool {
void FlushTextureCaches();
#endif // defined(__APPLE__)
// This generates a "rol" instruction with both Clang and GCC.
inline static std::size_t RotateLeft(std::size_t x, int n) {
return (x << n) | (x >> (std::numeric_limits<size_t>::digits - n));
}
struct BufferSpec {
BufferSpec(int w, int h, mediapipe::GpuBufferFormat f)
: width(w), height(h), format(f) {}
int width;
int height;
mediapipe::GpuBufferFormat format;
};
struct BufferSpecHash {
std::size_t operator()(const BufferSpec& spec) const {
// Width and height are expected to be smaller than half the width of
// size_t. We can combine them into a single integer, and then use
// std::hash, which is what go/hashing recommends for hashing numbers.
constexpr int kWidth = std::numeric_limits<size_t>::digits;
return std::hash<std::size_t>{}(
spec.width ^ RotateLeft(spec.height, kWidth / 2) ^
RotateLeft(static_cast<uint32_t>(spec.format), kWidth / 4));
}
};
private:
#if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
using SimplePool = std::shared_ptr<CvPixelBufferPoolWrapper>;
@@ -175,6 +151,35 @@ class GpuBufferMultiPool {
#endif // defined(__APPLE__)
};
#if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
class CvPixelBufferPoolWrapper {
public:
CvPixelBufferPoolWrapper(const GpuBufferMultiPool::BufferSpec& spec,
CFTimeInterval maxAge);
GpuBuffer GetBuffer(std::function<void(void)> flush);
int GetBufferCount() const { return count_; }
std::string GetDebugString() const;
void Flush();
private:
CFHolder<CVPixelBufferPoolRef> pool_;
int count_ = 0;
};
#endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
// BufferSpec equality operators
inline bool operator==(const GpuBufferMultiPool::BufferSpec& lhs,
const GpuBufferMultiPool::BufferSpec& rhs) {
return lhs.width == rhs.width && lhs.height == rhs.height &&
lhs.format == rhs.format;
}
inline bool operator!=(const GpuBufferMultiPool::BufferSpec& lhs,
const GpuBufferMultiPool::BufferSpec& rhs) {
return !operator==(lhs, rhs);
}
} // namespace mediapipe
#endif // MEDIAPIPE_GPU_GPU_BUFFER_MULTI_POOL_H_
mediapipe/gpu/gpu_shared_data_internal.cc
+5
View File
@@ -110,6 +110,7 @@ GpuResources::~GpuResources() {
std::string node_type = node->GetCalculatorState().CalculatorType();
std::string context_key;
#ifndef __EMSCRIPTEN__
// TODO Allow calculators to request a separate context.
// For now, white-list a few calculators to run in their own context.
bool gets_own_context = (node_type == "ImageFrameToGpuBufferCalculator") ||
@@ -126,6 +127,10 @@ GpuResources::~GpuResources() {
} else {
context_key = absl::StrCat("auto:", node_id);
}
#else
// On Emscripten we currently do not support multiple contexts.
context_key = SharedContextKey();
#endif // !__EMSCRIPTEN__
node_key_[node_id] = context_key;
ASSIGN_OR_RETURN(std::shared_ptr<GlContext> context,
mediapipe/graphs/face_effect/BUILD
+43
View File
@@ -0,0 +1,43 @@
# Copyright 2020 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
load(
"//mediapipe/framework/tool:mediapipe_graph.bzl",
"mediapipe_binary_graph",
)
licenses(["notice"])
package(default_visibility = ["//visibility:public"])
cc_library(
name = "face_effect_gpu_deps",
deps = [
"//mediapipe/calculators/core:flow_limiter_calculator",
"//mediapipe/calculators/core:gate_calculator",
"//mediapipe/calculators/core:immediate_mux_calculator",
"//mediapipe/calculators/image:image_properties_calculator",
"//mediapipe/graphs/face_effect/subgraphs:single_face_smooth_landmark_gpu",
"//mediapipe/modules/face_geometry",
"//mediapipe/modules/face_geometry:effect_renderer_calculator",
"//mediapipe/modules/face_geometry:env_generator_calculator",
],
)
mediapipe_binary_graph(
name = "face_effect_gpu_binary_graph",
graph = "face_effect_gpu.pbtxt",
output_name = "face_effect_gpu.binarypb",
deps = [":face_effect_gpu_deps"],
)
mediapipe/graphs/face_effect/data/BUILD
+36
View File
@@ -0,0 +1,36 @@
# Copyright 2020 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
load("//mediapipe/framework:encode_binary_proto.bzl", "encode_binary_proto")
licenses(["notice"])
package(default_visibility = ["//visibility:public"])
encode_binary_proto(
name = "glasses",
input = "glasses.pbtxt",
message_type = "mediapipe.face_geometry.Mesh3d",
output = "glasses.binarypb",
deps = [
"//mediapipe/modules/face_geometry/protos:mesh_3d_proto",
],
)
# `.pngblob` is used instead of `.png` to prevent iOS build from preprocessing the image.
# OpenCV is unable to read a PNG file preprocessed by the iOS build.
exports_files([
"facepaint.pngblob",
"glasses.pngblob",
])
mediapipe/graphs/face_effect/data/facepaint.pngblob
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 593 KiB

mediapipe/graphs/face_effect/data/glasses.pbtxt
+27815
View File
File diff suppressed because it is too large Load Diff
mediapipe/graphs/face_effect/data/glasses.pngblob
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 293 KiB

mediapipe/graphs/face_effect/face_effect_gpu.pbtxt
+145
View File
@@ -0,0 +1,145 @@
# MediaPipe graph that applies a face effect to the input video stream.
# GPU buffer. (GpuBuffer)
input_stream: "input_video"
# Boolean flag, which indicates whether the Facepaint effect is selected. (bool)
#
# If `true`, the Facepaint effect will be rendered.
# If `false`, the Glasses effect will be rendered.
input_stream: "is_facepaint_effect_selected"
# Output image with rendered results. (GpuBuffer)
output_stream: "output_video"
# A list of geometry data for a single detected face.
#
# NOTE: there will not be an output packet in this stream for this particular
# timestamp if none of faces detected.
#
# (std::vector<face_geometry::FaceGeometry>)
output_stream: "multi_face_geometry"
# Throttles the images flowing downstream for flow control. It passes through
# the very first incoming image unaltered, and waits for downstream nodes
# (calculators and subgraphs) in the graph to finish their tasks before it
# passes through another image. All images that come in while waiting are
# dropped, limiting the number of in-flight images in most part of the graph to
# 1. This prevents the downstream nodes from queuing up incoming images and data
# excessively, which leads to increased latency and memory usage, unwanted in
# real-time mobile applications. It also eliminates unnecessarily computation,
# e.g., the output produced by a node may get dropped downstream if the
# subsequent nodes are still busy processing previous inputs.
node {
calculator: "FlowLimiterCalculator"
input_stream: "input_video"
input_stream: "FINISHED:output_video"
input_stream_info: {
tag_index: "FINISHED"
back_edge: true
}
output_stream: "throttled_input_video"
}
# Generates an environment that describes the current virtual scene.
node {
calculator: "FaceGeometryEnvGeneratorCalculator"
output_side_packet: "ENVIRONMENT:environment"
node_options: {
[type.googleapis.com/mediapipe.FaceGeometryEnvGeneratorCalculatorOptions] {
environment: {
origin_point_location: TOP_LEFT_CORNER
perspective_camera: {
vertical_fov_degrees: 63.0 # 63 degrees
near: 1.0 # 1cm
far: 10000.0 # 100m
}
}
}
}
}
# Subgraph that detects a single face and corresponding landmarks. The landmarks
# are also "smoothed" to achieve better visual results.
node {
calculator: "SingleFaceSmoothLandmarkGpu"
input_stream: "IMAGE:throttled_input_video"
output_stream: "LANDMARKS:multi_face_landmarks"
}
# Extracts the throttled input video frame dimensions as a separate packet.
node {
calculator: "ImagePropertiesCalculator"
input_stream: "IMAGE_GPU:throttled_input_video"
output_stream: "SIZE:input_video_size"
}
# Subgraph that computes face geometry from landmarks for a single face.
node {
calculator: "FaceGeometry"
input_stream: "MULTI_FACE_LANDMARKS:multi_face_landmarks"
input_stream: "IMAGE_SIZE:input_video_size"
input_side_packet: "ENVIRONMENT:environment"
output_stream: "MULTI_FACE_GEOMETRY:multi_face_geometry"
}
# Decides whether to render the Facepaint effect based on the
# `is_facepaint_effect_selected` flag value.
node {
calculator: "GateCalculator"
input_stream: "throttled_input_video"
input_stream: "multi_face_geometry"
input_stream: "ALLOW:is_facepaint_effect_selected"
output_stream: "facepaint_effect_throttled_input_video"
output_stream: "facepaint_effect_multi_face_geometry"
}
# Renders the Facepaint effect.
node {
calculator: "FaceGeometryEffectRendererCalculator"
input_side_packet: "ENVIRONMENT:environment"
input_stream: "IMAGE_GPU:facepaint_effect_throttled_input_video"
input_stream: "MULTI_FACE_GEOMETRY:facepaint_effect_multi_face_geometry"
output_stream: "IMAGE_GPU:facepaint_effect_output_video"
node_options: {
[type.googleapis.com/mediapipe.FaceGeometryEffectRendererCalculatorOptions] {
effect_texture_path: "mediapipe/graphs/face_effect/data/facepaint.pngblob"
}
}
}
# Decides whether to render the Glasses effect based on the
# `is_facepaint_effect_selected` flag value.
node {
calculator: "GateCalculator"
input_stream: "throttled_input_video"
input_stream: "multi_face_geometry"
input_stream: "DISALLOW:is_facepaint_effect_selected"
output_stream: "glasses_effect_throttled_input_video"
output_stream: "glasses_effect_multi_face_geometry"
}
# Renders the Glasses effect.
node {
calculator: "FaceGeometryEffectRendererCalculator"
input_side_packet: "ENVIRONMENT:environment"
input_stream: "IMAGE_GPU:glasses_effect_throttled_input_video"
input_stream: "MULTI_FACE_GEOMETRY:glasses_effect_multi_face_geometry"
output_stream: "IMAGE_GPU:glasses_effect_output_video"
node_options: {
[type.googleapis.com/mediapipe.FaceGeometryEffectRendererCalculatorOptions] {
effect_texture_path: "mediapipe/graphs/face_effect/data/glasses.pngblob"
effect_mesh_3d_path: "mediapipe/graphs/face_effect/data/glasses.binarypb"
}
}
}
# Decides which of the Facepaint or the Glasses rendered results should be sent
# as the output GPU frame.
node {
calculator: "ImmediateMuxCalculator"
input_stream: "facepaint_effect_output_video"
input_stream: "glasses_effect_output_video"
output_stream: "output_video"
}
mediapipe/graphs/face_effect/subgraphs/BUILD
+36
View File
@@ -0,0 +1,36 @@
# Copyright 2020 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
load(
"//mediapipe/framework/tool:mediapipe_graph.bzl",
"mediapipe_simple_subgraph",
)
licenses(["notice"])
package(default_visibility = ["//visibility:public"])
mediapipe_simple_subgraph(
name = "single_face_smooth_landmark_gpu",
graph = "single_face_smooth_landmark_gpu.pbtxt",
register_as = "SingleFaceSmoothLandmarkGpu",
deps = [
"//mediapipe/calculators/core:concatenate_vector_calculator",
"//mediapipe/calculators/core:constant_side_packet_calculator",
"//mediapipe/calculators/core:split_vector_calculator",
"//mediapipe/calculators/image:image_properties_calculator",
"//mediapipe/calculators/util:landmarks_smoothing_calculator",
"//mediapipe/modules/face_landmark:face_landmark_front_gpu",
],
)
mediapipe/graphs/face_effect/subgraphs/single_face_smooth_landmark_gpu.pbtxt
+84
View File
@@ -0,0 +1,84 @@
# MediaPipe subgraph that detects a single face and corresponding landmarks on
# a input GPU image. The landmarks are also "smoothed" to achieve better visual
# results.
type: "SingleFaceSmoothLandmarkGpu"
# GPU image. (GpuBuffer)
input_stream: "IMAGE:input_image"
# Collection of detected/predicted faces, each represented as a list of face
# landmarks. However, the size of this collection is always 1 because of the
# single-face use in this graph. The decision to wrap the landmark list into a
# collection was made to simplify passing the result into the `FaceGeometry`
# subgraph. (std::vector<NormalizedLandmarkList>)
#
# NOTE: there will not be an output packet in the LANDMARKS stream for this
# particular timestamp if none of faces detected. However, the MediaPipe
# framework will internally inform the downstream calculators of the absence of
# this packet so that they don't wait for it unnecessarily.
output_stream: "LANDMARKS:multi_face_smooth_landmarks"
# Creates a packet to inform the `FaceLandmarkFrontGpu` subgraph to detect at
# most 1 face.
node {
calculator: "ConstantSidePacketCalculator"
output_side_packet: "PACKET:num_faces"
node_options: {
[type.googleapis.com/mediapipe.ConstantSidePacketCalculatorOptions]: {
packet { int_value: 1 }
}
}
}
# Subgraph that detects faces and corresponding landmarks.
node {
calculator: "FaceLandmarkFrontGpu"
input_stream: "IMAGE:input_image"
input_side_packet: "NUM_FACES:num_faces"
output_stream: "LANDMARKS:multi_face_landmarks"
}
# Extracts the detected face landmark list from a collection.
node {
calculator: "SplitNormalizedLandmarkListVectorCalculator"
input_stream: "multi_face_landmarks"
output_stream: "face_landmarks"
node_options: {
[type.googleapis.com/mediapipe.SplitVectorCalculatorOptions] {
ranges: { begin: 0 end: 1 }
element_only: true
}
}
}
# Extracts the input image frame dimensions as a separate packet.
node {
calculator: "ImagePropertiesCalculator"
input_stream: "IMAGE_GPU:input_image"
output_stream: "SIZE:input_image_size"
}
# Applies smoothing to the single face landmarks.
node {
calculator: "LandmarksSmoothingCalculator"
input_stream: "NORM_LANDMARKS:face_landmarks"
input_stream: "IMAGE_SIZE:input_image_size"
output_stream: "NORM_FILTERED_LANDMARKS:face_smooth_landmarks"
node_options: {
[type.googleapis.com/mediapipe.LandmarksSmoothingCalculatorOptions] {
velocity_filter: {
window_size: 5
velocity_scale: 20.0
}
}
}
}
# Puts the single face smooth landmarks back into a collection to simplify
# passing the result into the `FaceGeometry` subgraph.
node {
calculator: "ConcatenateLandmarListVectorCalculator"
input_stream: "face_smooth_landmarks"
output_stream: "multi_face_smooth_landmarks"
}
mediapipe/graphs/object_detection_3d/calculators/BUILD
+1 -1
View File
@@ -16,7 +16,7 @@ load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library"
licenses(["notice"])
package(default_visibility = ["//visibility:private"])
package(default_visibility = ["//visibility:public"])
proto_library(
name = "object_proto",
mediapipe/graphs/pose_tracking/BUILD
+2 -2
View File
@@ -26,7 +26,7 @@ cc_library(
deps = [
"//mediapipe/calculators/core:flow_limiter_calculator",
"//mediapipe/calculators/image:image_properties_calculator",
"//mediapipe/graphs/pose_tracking/calculators:landmarks_smoothing_calculator",
"//mediapipe/calculators/util:landmarks_smoothing_calculator",
"//mediapipe/graphs/pose_tracking/subgraphs:upper_body_pose_renderer_gpu",
"//mediapipe/modules/pose_landmark:pose_landmark_upper_body_gpu",
],
@@ -44,7 +44,7 @@ cc_library(
deps = [
"//mediapipe/calculators/core:flow_limiter_calculator",
"//mediapipe/calculators/image:image_properties_calculator",
"//mediapipe/graphs/pose_tracking/calculators:landmarks_smoothing_calculator",
"//mediapipe/calculators/util:landmarks_smoothing_calculator",
"//mediapipe/graphs/pose_tracking/subgraphs:upper_body_pose_renderer_cpu",
"//mediapipe/modules/pose_landmark:pose_landmark_upper_body_cpu",
],
mediapipe/java/com/google/mediapipe/components/FrameProcessor.java
+9 -6
View File
@@ -233,19 +233,20 @@ public class FrameProcessor implements TextureFrameProcessor, AudioDataProcessor
*
* @param inputStream the graph input stream that will receive input audio samples.
* @param outputStream the output stream from which output audio samples will be produced.
* @param numChannels the number of audio channels in the input audio stream.
* @param numInputChannels the number of audio channels in the input audio stream.
* @param numOutputChannels the number of audio channels in the output audio stream. If there is
* no output stream, set this to zero.
* @param audioSampleRateInHz the sample rate for audio samples in hertz (Hz).
*/
public void addAudioStreams(
@Nullable String inputStream,
@Nullable String outputStream,
int numChannels,
int numInputChannels,
int numOutputChannels,
double audioSampleRateInHz) {
audioInputStream = inputStream;
audioOutputStream = outputStream;
numAudioChannels = numChannels;
int audioChannelMask =
numAudioChannels == 2 ? AudioFormat.CHANNEL_IN_STEREO : AudioFormat.CHANNEL_IN_MONO;
numAudioChannels = numInputChannels;
audioSampleRate = audioSampleRateInHz;
if (audioInputStream != null) {
@@ -254,11 +255,13 @@ public class FrameProcessor implements TextureFrameProcessor, AudioDataProcessor
}
if (audioOutputStream != null) {
int outputAudioChannelMask =
numOutputChannels == 2 ? AudioFormat.CHANNEL_IN_STEREO : AudioFormat.CHANNEL_IN_MONO;
AudioFormat audioFormat =
new AudioFormat.Builder()
.setEncoding(AUDIO_ENCODING)
.setSampleRate((int) audioSampleRate)
.setChannelMask(audioChannelMask)
.setChannelMask(outputAudioChannelMask)
.build();
mediapipeGraph.addPacketCallback(
audioOutputStream,
mediapipe/java/com/google/mediapipe/mediapipe_aar.bzl
+14
View File
@@ -1,3 +1,17 @@
# Copyright 2019-2020 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Generate MediaPipe AAR including different variants of .so in jni folder.
Usage:
mediapipe/models/README.md
+1 -69
View File
@@ -1,69 +1 @@
## MediaPipe Models
### Face Detection
* For front-facing/selfie cameras: [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/face_detection_front.tflite)
* For back-facing cameras: [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/face_detection_back.tflite)
* [Model page](https://sites.google.com/corp/view/perception-cv4arvr/blazeface)
* Paper: ["BlazeFace: Sub-millisecond Neural Face Detection on Mobile GPUs"](https://arxiv.org/abs/1907.05047)
* [Model card](https://mediapipe.page.link/blazeface-mc)
### Face Mesh
* Face detection: [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/face_detection_front.tflite) (see above)
* 3D face landmarks: [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/face_landmark.tflite), [TF.js model](https://tfhub.dev/mediapipe/facemesh/1)
* [Model page](https://sites.google.com/corp/view/perception-cv4arvr/facemesh)
* Paper: ["Real-time Facial Surface Geometry from Monocular Video on Mobile GPUs"](https://arxiv.org/abs/1907.06724)
* [Google AI Blog post](https://ai.googleblog.com/2019/03/real-time-ar-self-expression-with.html)
* [TensorFlow Blog post](https://blog.tensorflow.org/2020/03/face-and-hand-tracking-in-browser-with-mediapipe-and-tensorflowjs.html)
* [Model card](https://mediapipe.page.link/facemesh-mc)
### Hand Detection and Tracking
* Palm detection: [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/palm_detection.tflite), [TF.js model](https://tfhub.dev/mediapipe/handdetector/1)
* 3D hand landmarks: [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/hand_landmark.tflite), [TF.js model](https://tfhub.dev/mediapipe/handskeleton/1)
* [Google AI Blog post](https://ai.googleblog.com/2019/08/on-device-real-time-hand-tracking-with.html)
* [TensorFlow Blog post](https://blog.tensorflow.org/2020/03/face-and-hand-tracking-in-browser-with-mediapipe-and-tensorflowjs.html)
* [Model card](https://mediapipe.page.link/handmc)
### Iris
* Iris landmarks:
[TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/iris_landmark.tflite)
* Paper:
[Real-time Pupil Tracking from Monocular Video for Digital Puppetry](https://arxiv.org/abs/2006.11341)
([presentation](https://youtu.be/cIhXkiiapQI))
* Google AI Blog:
[MediaPipe Iris: Real-time Eye Tracking and Depth Estimation](https://ai.googleblog.com/2020/08/mediapipe-iris-real-time-iris-tracking.html)
* [Model card](https://mediapipe.page.link/iris-mc)
### Pose
* Pose detection:
[TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/pose_detection/pose_detection.tflite)
* Upper-body pose landmarks:
[TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/pose_landmark/pose_landmark_upper_body.tflite)
* Paper:
[BlazePose: On-device Real-time Body Pose Tracking](https://arxiv.org/abs/2006.10204)
([presentation](https://youtu.be/YPpUOTRn5tA))
* Google AI Blog:
[BlazePose - On-device Real-time Body Pose Tracking](https://ai.googleblog.com/2020/08/on-device-real-time-body-pose-tracking.html)
* [Model card](https://mediapipe.page.link/blazepose-mc)
### Hair Segmentation
* [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/hair_segmentation.tflite)
* [Model page](https://sites.google.com/corp/view/perception-cv4arvr/hair-segmentation)
* Paper: ["Real-time Hair segmentation and recoloring on Mobile GPUs"](https://arxiv.org/abs/1907.06740)
* [Model card](https://mediapipe.page.link/hairsegmentation-mc)
### Objectron (3D Object Detection)
* Shoes: [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_3d_sneakers.tflite)
* Chairs: [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_3d_chair.tflite)
* [Google AI Blog post](https://ai.googleblog.com/2020/03/real-time-3d-object-detection-on-mobile.html)
### Object Detection
* [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/ssdlite_object_detection.tflite)
* See [here](object_detection_saved_model/README.md) for model details.
### KNIFT (Keypoint Neural Invariant Feature Transform)
* Up to 200 keypoints: [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/knift_float.tflite)
* Up to 400 keypoints: [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/knift_float_400.tflite)
* Up to 1000 keypoints: [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/knift_float_1k.tflite)
* [Google Developers Blog post](https://developers.googleblog.com/2020/04/mediapipe-knift-template-based-feature-matching.html)
* [Model card](https://mediapipe.page.link/knift-mc)
Please see https://solutions.mediapipe.dev/models for more description and model cards.
mediapipe/modules/face_geometry/BUILD
+116
View File
@@ -0,0 +1,116 @@
# Copyright 2020 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library")
load("//mediapipe/framework/tool:mediapipe_graph.bzl", "mediapipe_simple_subgraph")
licenses(["notice"])
package(default_visibility = ["//visibility:public"])
mediapipe_simple_subgraph(
name = "face_geometry",
graph = "face_geometry.pbtxt",
register_as = "FaceGeometry",
deps = [
":geometry_pipeline_calculator",
],
)
mediapipe_proto_library(
name = "effect_renderer_calculator_proto",
srcs = ["effect_renderer_calculator.proto"],
deps = [
"//mediapipe/framework:calculator_options_proto",
],
)
cc_library(
name = "effect_renderer_calculator",
srcs = ["effect_renderer_calculator.cc"],
deps = [
":effect_renderer_calculator_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/formats:image_frame_opencv",
"//mediapipe/framework/port:opencv_core",
"//mediapipe/framework/port:opencv_imgcodecs",
"//mediapipe/framework/port:opencv_imgproc",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"//mediapipe/framework/port:statusor",
"//mediapipe/gpu:gl_calculator_helper",
"//mediapipe/gpu:gpu_buffer",
"//mediapipe/modules/face_geometry/libs:effect_renderer",
"//mediapipe/modules/face_geometry/libs:validation_utils",
"//mediapipe/modules/face_geometry/protos:environment_cc_proto",
"//mediapipe/modules/face_geometry/protos:face_geometry_cc_proto",
"//mediapipe/modules/face_geometry/protos:mesh_3d_cc_proto",
"//mediapipe/util:resource_util",
"@com_google_absl//absl/types:optional",
],
alwayslink = 1,
)
mediapipe_proto_library(
name = "env_generator_calculator_proto",
srcs = ["env_generator_calculator.proto"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/modules/face_geometry/protos:environment_proto",
],
)
cc_library(
name = "env_generator_calculator",
srcs = ["env_generator_calculator.cc"],
deps = [
":env_generator_calculator_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/port:status",
"//mediapipe/modules/face_geometry/libs:validation_utils",
"//mediapipe/modules/face_geometry/protos:environment_cc_proto",
],
alwayslink = 1,
)
mediapipe_proto_library(
name = "geometry_pipeline_calculator_proto",
srcs = ["geometry_pipeline_calculator.proto"],
deps = [
"//mediapipe/framework:calculator_options_proto",
],
)
cc_library(
name = "geometry_pipeline_calculator",
srcs = ["geometry_pipeline_calculator.cc"],
deps = [
":geometry_pipeline_calculator_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/formats:landmark_cc_proto",
"//mediapipe/framework/port:logging",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"//mediapipe/framework/port:statusor",
"//mediapipe/modules/face_geometry/libs:geometry_pipeline",
"//mediapipe/modules/face_geometry/libs:validation_utils",
"//mediapipe/modules/face_geometry/protos:environment_cc_proto",
"//mediapipe/modules/face_geometry/protos:face_geometry_cc_proto",
"//mediapipe/modules/face_geometry/protos:geometry_pipeline_metadata_cc_proto",
"//mediapipe/util:resource_util",
"@com_google_absl//absl/memory",
],
alwayslink = 1,
)
mediapipe/modules/face_geometry/README.md
+18
View File
@@ -0,0 +1,18 @@
# face_geometry
Protos|Details
:--- | :---
[`face_geometry.Environment`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/protos/environment.proto)| Describes an environment; includes the camera frame origin point location as well as virtual camera parameters.
[`face_geometry.GeometryPipelineMetadata`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/protos/geometry_pipeline_metadata.proto)| Describes metadata needed to estimate face geometry based on the face landmark module result.
[`face_geometry.FaceGeometry`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/protos/face_geometry.proto)| Describes geometry data for a single face; includes a face mesh surface and a face pose in a given environment.
[`face_geometry.Mesh3d`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/protos/mesh_3d.proto)| Describes a 3D mesh surface.
Calculators|Details
:--- | :---
[`FaceGeometryEnvGeneratorCalculator`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/env_generator_calculator.cc)| Generates an environment that describes a virtual scene.
[`FaceGeometryPipelineCalculator`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/geometry_pipeline_calculator.cc)| Extracts face geometry for multiple faces from a vector of landmark lists.
[`FaceGeometryEffectRendererCalculator`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/effect_renderer_calculator.cc)| Renders a face effect.
Subgraphs|Details
:--- | :---
[`FaceGeometry`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/face_geometry.pbtxt)| Extracts face geometry from landmarks for multiple faces.
mediapipe/modules/face_geometry/data/BUILD
+37
View File
@@ -0,0 +1,37 @@
# Copyright 2020 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
load("//mediapipe/framework:encode_binary_proto.bzl", "encode_binary_proto")
licenses(["notice"])
package(default_visibility = ["//visibility:public"])
encode_binary_proto(
name = "geometry_pipeline_metadata",
input = "geometry_pipeline_metadata.pbtxt",
message_type = "mediapipe.face_geometry.GeometryPipelineMetadata",
output = "geometry_pipeline_metadata.binarypb",
deps = [
"//mediapipe/modules/face_geometry/protos:geometry_pipeline_metadata_proto",
],
)
# These canonical face model files are not meant to be used in runtime, but rather for asset
# creation and/or reference.
exports_files([
"canonical_face_model.fbx",
"canonical_face_model.obj",
"canonical_face_model_uv_visualization.png",
])
mediapipe/modules/face_geometry/data/canonical_face_model.fbx
BIN
View File
Binary file not shown.
mediapipe/modules/face_geometry/data/canonical_face_model.obj
+1834
View File
File diff suppressed because it is too large Load Diff
mediapipe/modules/face_geometry/data/canonical_face_model_uv_visualization.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 731 KiB

mediapipe/modules/face_geometry/data/geometry_pipeline_metadata.pbtxt
+5085
View File
File diff suppressed because it is too large Load Diff
mediapipe/modules/face_geometry/effect_renderer_calculator.cc
+284
View File
@@ -0,0 +1,284 @@
// Copyright 2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <memory>
#include <string>
#include <vector>
#include "absl/types/optional.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/formats/image_frame_opencv.h"
#include "mediapipe/framework/port/opencv_core_inc.h" // NOTYPO
#include "mediapipe/framework/port/opencv_imgcodecs_inc.h" // NOTYPO
#include "mediapipe/framework/port/opencv_imgproc_inc.h" // NOTYPO
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/framework/port/status_macros.h"
#include "mediapipe/framework/port/statusor.h"
#include "mediapipe/gpu/gl_calculator_helper.h"
#include "mediapipe/gpu/gpu_buffer.h"
#include "mediapipe/modules/face_geometry/effect_renderer_calculator.pb.h"
#include "mediapipe/modules/face_geometry/libs/effect_renderer.h"
#include "mediapipe/modules/face_geometry/libs/validation_utils.h"
#include "mediapipe/modules/face_geometry/protos/environment.pb.h"
#include "mediapipe/modules/face_geometry/protos/face_geometry.pb.h"
#include "mediapipe/modules/face_geometry/protos/mesh_3d.pb.h"
#include "mediapipe/util/resource_util.h"
namespace mediapipe {
namespace {
static constexpr char kEnvironmentTag[] = "ENVIRONMENT";
static constexpr char kImageGpuTag[] = "IMAGE_GPU";
static constexpr char kMultiFaceGeometryTag[] = "MULTI_FACE_GEOMETRY";
// A calculator that renders a visual effect for multiple faces.
//
// Inputs:
// IMAGE_GPU (`GpuBuffer`, required):
// A buffer containing input image.
//
// MULTI_FACE_GEOMETRY (`std::vector<face_geometry::FaceGeometry>`, optional):
// A vector of face geometry data.
//
// If absent, the input GPU buffer is copied over into the output GPU buffer
// without any effect being rendered.
//
// Input side packets:
// ENVIRONMENT (`face_geometry::Environment`, required)
// Describes an environment; includes the camera frame origin point location
// as well as virtual camera parameters.
//
// Output:
// IMAGE_GPU (`GpuBuffer`, required):
// A buffer with a visual effect being rendered for multiple faces.
//
// Options:
// effect_texture_path (`string`, required):
// Defines a path for the visual effect texture file. The effect texture is
// later rendered on top of the effect mesh.
//
// The texture file format must be supported by the OpenCV image decoder. It
// must also define either an RGB or an RGBA texture.
//
// effect_mesh_3d_path (`string`, optional):
// Defines a path for the visual effect mesh 3D file. The effect mesh is
// later "attached" to the face and is driven by the face pose
// transformation matrix.
//
// The mesh 3D file format must be the binary `face_geometry.Mesh3d` proto.
//
// If is not present, the runtime face mesh will be used as the effect mesh
// - this mode is handy for facepaint effects.
//
class EffectRendererCalculator : public CalculatorBase {
public:
static mediapipe::Status GetContract(CalculatorContract* cc) {
MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc))
<< "Failed to update contract for the GPU helper!";
cc->InputSidePackets()
.Tag(kEnvironmentTag)
.Set<face_geometry::Environment>();
cc->Inputs().Tag(kImageGpuTag).Set<GpuBuffer>();
cc->Inputs()
.Tag(kMultiFaceGeometryTag)
.Set<std::vector<face_geometry::FaceGeometry>>();
cc->Outputs().Tag(kImageGpuTag).Set<GpuBuffer>();
return mediapipe::GlCalculatorHelper::UpdateContract(cc);
}
mediapipe::Status Open(CalculatorContext* cc) override {
cc->SetOffset(mediapipe::TimestampDiff(0));
MP_RETURN_IF_ERROR(gpu_helper_.Open(cc))
<< "Failed to open the GPU helper!";
return gpu_helper_.RunInGlContext([&]() -> mediapipe::Status {
const auto& options =
cc->Options<FaceGeometryEffectRendererCalculatorOptions>();
const auto& environment = cc->InputSidePackets()
.Tag(kEnvironmentTag)
.Get<face_geometry::Environment>();
MP_RETURN_IF_ERROR(face_geometry::ValidateEnvironment(environment))
<< "Invalid environment!";
absl::optional<face_geometry::Mesh3d> effect_mesh_3d;
if (options.has_effect_mesh_3d_path()) {
ASSIGN_OR_RETURN(effect_mesh_3d,
ReadMesh3dFromFile(options.effect_mesh_3d_path()),
_ << "Failed to read the effect 3D mesh from file!");
MP_RETURN_IF_ERROR(face_geometry::ValidateMesh3d(*effect_mesh_3d))
<< "Invalid effect 3D mesh!";
}
ASSIGN_OR_RETURN(ImageFrame effect_texture,
ReadTextureFromFile(options.effect_texture_path()),
_ << "Failed to read the effect texture from file!");
ASSIGN_OR_RETURN(effect_renderer_,
CreateEffectRenderer(environment, effect_mesh_3d,
std::move(effect_texture)),
_ << "Failed to create the effect renderer!");
return mediapipe::OkStatus();
});
}
mediapipe::Status Process(CalculatorContext* cc) override {
// The `IMAGE_GPU` stream is required to have a non-empty packet. In case
// this requirement is not met, there's nothing to be processed at the
// current timestamp.
if (cc->Inputs().Tag(kImageGpuTag).IsEmpty()) {
return mediapipe::OkStatus();
}
return gpu_helper_.RunInGlContext([this, cc]() -> mediapipe::Status {
const auto& input_gpu_buffer =
cc->Inputs().Tag(kImageGpuTag).Get<GpuBuffer>();
GlTexture input_gl_texture =
gpu_helper_.CreateSourceTexture(input_gpu_buffer);
GlTexture output_gl_texture = gpu_helper_.CreateDestinationTexture(
input_gl_texture.width(), input_gl_texture.height());
std::vector<face_geometry::FaceGeometry> empty_multi_face_geometry;
const auto& multi_face_geometry =
cc->Inputs().Tag(kMultiFaceGeometryTag).IsEmpty()
? empty_multi_face_geometry
: cc->Inputs()
.Tag(kMultiFaceGeometryTag)
.Get<std::vector<face_geometry::FaceGeometry>>();
// Validate input multi face geometry data.
for (const face_geometry::FaceGeometry& face_geometry :
multi_face_geometry) {
MP_RETURN_IF_ERROR(face_geometry::ValidateFaceGeometry(face_geometry))
<< "Invalid face geometry!";
}
MP_RETURN_IF_ERROR(effect_renderer_->RenderEffect(
multi_face_geometry, input_gl_texture.width(),
input_gl_texture.height(), input_gl_texture.target(),
input_gl_texture.name(), output_gl_texture.target(),
output_gl_texture.name()))
<< "Failed to render the effect!";
std::unique_ptr<GpuBuffer> output_gpu_buffer =
output_gl_texture.GetFrame<GpuBuffer>();
cc->Outputs()
.Tag(kImageGpuTag)
.AddPacket(mediapipe::Adopt<GpuBuffer>(output_gpu_buffer.release())
.At(cc->InputTimestamp()));
output_gl_texture.Release();
input_gl_texture.Release();
return mediapipe::OkStatus();
});
}
~EffectRendererCalculator() {
gpu_helper_.RunInGlContext([this]() { effect_renderer_.reset(); });
}
private:
static mediapipe::StatusOr<ImageFrame> ReadTextureFromFile(
const std::string& texture_path) {
ASSIGN_OR_RETURN(std::string texture_blob,
ReadContentBlobFromFile(texture_path),
_ << "Failed to read texture blob from file!");
// Use OpenCV image decoding functionality to finish reading the texture.
std::vector<char> texture_blob_vector(texture_blob.begin(),
texture_blob.end());
cv::Mat decoded_mat =
cv::imdecode(texture_blob_vector, cv::IMREAD_UNCHANGED);
RET_CHECK(decoded_mat.type() == CV_8UC3 || decoded_mat.type() == CV_8UC4)
<< "Texture must have `char` as the underlying type and "
"must have either 3 or 4 channels!";
ImageFormat::Format image_format = ImageFormat::UNKNOWN;
cv::Mat output_mat;
switch (decoded_mat.channels()) {
case 3:
image_format = ImageFormat::SRGB;
cv::cvtColor(decoded_mat, output_mat, cv::COLOR_BGR2RGB);
break;
case 4:
image_format = ImageFormat::SRGBA;
cv::cvtColor(decoded_mat, output_mat, cv::COLOR_BGRA2RGBA);
break;
default:
RET_CHECK_FAIL()
<< "Unexpected number of channels; expected 3 or 4, got "
<< decoded_mat.channels() << "!";
}
ImageFrame output_image_frame(image_format, output_mat.size().width,
output_mat.size().height,
ImageFrame::kGlDefaultAlignmentBoundary);
output_mat.copyTo(formats::MatView(&output_image_frame));
return output_image_frame;
}
static mediapipe::StatusOr<face_geometry::Mesh3d> ReadMesh3dFromFile(
const std::string& mesh_3d_path) {
ASSIGN_OR_RETURN(std::string mesh_3d_blob,
ReadContentBlobFromFile(mesh_3d_path),
_ << "Failed to read mesh 3D blob from file!");
face_geometry::Mesh3d mesh_3d;
RET_CHECK(mesh_3d.ParseFromString(mesh_3d_blob))
<< "Failed to parse a mesh 3D proto from a binary blob!";
return mesh_3d;
}
static mediapipe::StatusOr<std::string> ReadContentBlobFromFile(
const std::string& unresolved_path) {
ASSIGN_OR_RETURN(std::string resolved_path,
mediapipe::PathToResourceAsFile(unresolved_path),
_ << "Failed to resolve path! Path = " << unresolved_path);
std::string content_blob;
MP_RETURN_IF_ERROR(
mediapipe::GetResourceContents(resolved_path, &content_blob))
<< "Failed to read content blob! Resolved path = " << resolved_path;
return content_blob;
}
mediapipe::GlCalculatorHelper gpu_helper_;
std::unique_ptr<face_geometry::EffectRenderer> effect_renderer_;
};
} // namespace
using FaceGeometryEffectRendererCalculator = EffectRendererCalculator;
REGISTER_CALCULATOR(FaceGeometryEffectRendererCalculator);
} // namespace mediapipe
mediapipe/modules/face_geometry/effect_renderer_calculator.proto
+46
View File
@@ -0,0 +1,46 @@
// Copyright 2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto2";
package mediapipe;
import "mediapipe/framework/calculator_options.proto";
message FaceGeometryEffectRendererCalculatorOptions {
extend CalculatorOptions {
optional FaceGeometryEffectRendererCalculatorOptions ext = 323693808;
}
// Defines a path for the visual effect texture file. The effect texture is
// later rendered on top of the effect mesh.
//
// Please be aware about the difference between the CPU texture memory layout
// and the GPU texture sampler coordinate space. This renderer follows
// conventions discussed here: https://open.gl/textures
//
// The texture file format must be supported by the OpenCV image decoder. It
// must also define either an RGB or an RGBA texture.
optional string effect_texture_path = 1;
// Defines a path for the visual effect mesh 3D file. The effect mesh is later
// "attached" to the face and is driven by the face pose transformation
// matrix.
//
// The mesh 3D file format must be the binary `face_system.Mesh3d` proto.
//
// If is not present, the runtime face mesh will be used as the effect mesh
// - this mode is handy for facepaint effects.
optional string effect_mesh_3d_path = 2;
}
mediapipe/modules/face_geometry/env_generator_calculator.cc
+81
View File
@@ -0,0 +1,81 @@
// Copyright 2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/framework/port/status_macros.h"
#include "mediapipe/modules/face_geometry/env_generator_calculator.pb.h"
#include "mediapipe/modules/face_geometry/libs/validation_utils.h"
#include "mediapipe/modules/face_geometry/protos/environment.pb.h"
namespace mediapipe {
namespace {
static constexpr char kEnvironmentTag[] = "ENVIRONMENT";
// A calculator that generates an environment, which describes a virtual scene.
//
// Output side packets:
// ENVIRONMENT (`face_geometry::Environment`, required)
// Describes an environment; includes the camera frame origin point location
// as well as virtual camera parameters.
//
// Options:
// environment (`face_geometry.Environment`, required):
// Defines an environment to be packed as the output side packet.
//
// Must be valid (for details, please refer to the proto message definition
// comments and/or `modules/face_geometry/libs/validation_utils.h/cc`)
//
class EnvGeneratorCalculator : public CalculatorBase {
public:
static mediapipe::Status GetContract(CalculatorContract* cc) {
cc->OutputSidePackets()
.Tag(kEnvironmentTag)
.Set<face_geometry::Environment>();
return mediapipe::OkStatus();
}
mediapipe::Status Open(CalculatorContext* cc) override {
cc->SetOffset(mediapipe::TimestampDiff(0));
const face_geometry::Environment& environment =
cc->Options<FaceGeometryEnvGeneratorCalculatorOptions>().environment();
MP_RETURN_IF_ERROR(face_geometry::ValidateEnvironment(environment))
<< "Invalid environment!";
cc->OutputSidePackets()
.Tag(kEnvironmentTag)
.Set(mediapipe::MakePacket<face_geometry::Environment>(environment));
return mediapipe::OkStatus();
}
mediapipe::Status Process(CalculatorContext* cc) override {
return mediapipe::OkStatus();
}
mediapipe::Status Close(CalculatorContext* cc) override {
return mediapipe::OkStatus();
}
};
} // namespace
using FaceGeometryEnvGeneratorCalculator = EnvGeneratorCalculator;
REGISTER_CALCULATOR(FaceGeometryEnvGeneratorCalculator);
} // namespace mediapipe
mediapipe/modules/face_geometry/env_generator_calculator.proto
+32
View File
@@ -0,0 +1,32 @@
// Copyright 2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto2";
package mediapipe;
import "mediapipe/framework/calculator_options.proto";
import "mediapipe/modules/face_geometry/protos/environment.proto";
message FaceGeometryEnvGeneratorCalculatorOptions {
extend CalculatorOptions {
optional FaceGeometryEnvGeneratorCalculatorOptions ext = 323693810;
}
// Defines an environment to be packed as the output side packet.
//
// Must be valid (for details, please refer to the proto message definition
// comments and/or `modules/face_geometry/libs/validation_utils.h/cc`)
optional face_geometry.Environment environment = 1;
}
mediapipe/modules/face_geometry/face_geometry.pbtxt
+52
View File
@@ -0,0 +1,52 @@
# MediaPipe graph to extract face geometry from landmarks for multiple faces.
#
# It is required that "geometry_pipeline_metadata.binarypb" is available at
# "mediapipe/modules/face_geometry/data/geometry_pipeline_metadata.binarypb"
# path during execution.
#
# EXAMPLE:
# node {
# calculator: "FaceGeometry"
# input_stream: "IMAGE_SIZE:image_size"
# input_stream: "MULTI_FACE_LANDMARKS:multi_face_landmarks"
# input_side_packet: "ENVIRONMENT:environment"
# output_stream: "MULTI_FACE_GEOMETRY:multi_face_geometry"
# }
type: "FaceGeometry"
# The size of the input frame. The first element of the pair is the frame width;
# the other one is the frame height.
#
# The face landmarks should have been detected on a frame with the same
# ratio. If used as-is, the resulting face geometry visualization should be
# happening on a frame with the same ratio as well.
#
# (std::pair<int, int>)
input_stream: "IMAGE_SIZE:image_size"
# Collection of detected/predicted faces, each represented as a list of face
# landmarks. (std::vector<NormalizedLandmarkList>)
input_stream: "MULTI_FACE_LANDMARKS:multi_face_landmarks"
# Environment that describes the current virtual scene.
# (face_geometry::Environment)
input_side_packet: "ENVIRONMENT:environment"
# A list of geometry data for each detected face.
# (std::vector<face_geometry::FaceGeometry>)
output_stream: "MULTI_FACE_GEOMETRY:multi_face_geometry"
# Extracts face geometry for multiple faces from a vector of landmark lists.
node {
calculator: "FaceGeometryPipelineCalculator"
input_side_packet: "ENVIRONMENT:environment"
input_stream: "IMAGE_SIZE:image_size"
input_stream: "MULTI_FACE_LANDMARKS:multi_face_landmarks"
output_stream: "MULTI_FACE_GEOMETRY:multi_face_geometry"
options: {
[mediapipe.FaceGeometryPipelineCalculatorOptions.ext] {
metadata_path: "mediapipe/modules/face_geometry/data/geometry_pipeline_metadata.binarypb"
}
}
}
mediapipe/modules/face_geometry/geometry_pipeline_calculator.cc
+197
View File
@@ -0,0 +1,197 @@
// Copyright 2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <memory>
#include <string>
#include <utility>
#include <vector>
#include "absl/memory/memory.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/landmark.pb.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/framework/port/status_macros.h"
#include "mediapipe/framework/port/statusor.h"
#include "mediapipe/modules/face_geometry/geometry_pipeline_calculator.pb.h"
#include "mediapipe/modules/face_geometry/libs/geometry_pipeline.h"
#include "mediapipe/modules/face_geometry/libs/validation_utils.h"
#include "mediapipe/modules/face_geometry/protos/environment.pb.h"
#include "mediapipe/modules/face_geometry/protos/face_geometry.pb.h"
#include "mediapipe/modules/face_geometry/protos/geometry_pipeline_metadata.pb.h"
#include "mediapipe/util/resource_util.h"
namespace mediapipe {
namespace {
static constexpr char kEnvironmentTag[] = "ENVIRONMENT";
static constexpr char kImageSizeTag[] = "IMAGE_SIZE";
static constexpr char kMultiFaceGeometryTag[] = "MULTI_FACE_GEOMETRY";
static constexpr char kMultiFaceLandmarksTag[] = "MULTI_FACE_LANDMARKS";
// A calculator that renders a visual effect for multiple faces.
//
// Inputs:
// IMAGE_SIZE (`std::pair<int, int>`, required):
// The size of the current frame. The first element of the pair is the frame
// width; the other one is the frame height.
//
// The face landmarks should have been detected on a frame with the same
// ratio. If used as-is, the resulting face geometry visualization should be
// happening on a frame with the same ratio as well.
//
// MULTI_FACE_LANDMARKS (`std::vector<NormalizedLandmarkList>`, required):
// A vector of face landmark lists.
//
// Input side packets:
// ENVIRONMENT (`face_geometry::Environment`, required)
// Describes an environment; includes the camera frame origin point location
// as well as virtual camera parameters.
//
// Output:
// MULTI_FACE_GEOMETRY (`std::vector<face_geometry::FaceGeometry>`, required):
// A vector of face geometry data.
//
// Options:
// metadata_path (`string`, optional):
// Defines a path for the geometry pipeline metadata file.
//
// The geometry pipeline metadata file format must be the binary
// `face_geometry.GeometryPipelineMetadata` proto.
//
class GeometryPipelineCalculator : public CalculatorBase {
public:
static mediapipe::Status GetContract(CalculatorContract* cc) {
cc->InputSidePackets()
.Tag(kEnvironmentTag)
.Set<face_geometry::Environment>();
cc->Inputs().Tag(kImageSizeTag).Set<std::pair<int, int>>();
cc->Inputs()
.Tag(kMultiFaceLandmarksTag)
.Set<std::vector<NormalizedLandmarkList>>();
cc->Outputs()
.Tag(kMultiFaceGeometryTag)
.Set<std::vector<face_geometry::FaceGeometry>>();
return mediapipe::OkStatus();
}
mediapipe::Status Open(CalculatorContext* cc) override {
cc->SetOffset(mediapipe::TimestampDiff(0));
const auto& options = cc->Options<FaceGeometryPipelineCalculatorOptions>();
ASSIGN_OR_RETURN(
face_geometry::GeometryPipelineMetadata metadata,
ReadMetadataFromFile(options.metadata_path()),
_ << "Failed to read the geometry pipeline metadata from file!");
MP_RETURN_IF_ERROR(
face_geometry::ValidateGeometryPipelineMetadata(metadata))
<< "Invalid geometry pipeline metadata!";
const face_geometry::Environment& environment =
cc->InputSidePackets()
.Tag(kEnvironmentTag)
.Get<face_geometry::Environment>();
MP_RETURN_IF_ERROR(face_geometry::ValidateEnvironment(environment))
<< "Invalid environment!";
ASSIGN_OR_RETURN(
geometry_pipeline_,
face_geometry::CreateGeometryPipeline(environment, metadata),
_ << "Failed to create a geometry pipeline!");
return mediapipe::OkStatus();
}
mediapipe::Status Process(CalculatorContext* cc) override {
// Both the `IMAGE_SIZE` and the `MULTI_FACE_LANDMARKS` streams are required
// to have a non-empty packet. In case this requirement is not met, there's
// nothing to be processed at the current timestamp.
if (cc->Inputs().Tag(kImageSizeTag).IsEmpty() ||
cc->Inputs().Tag(kMultiFaceLandmarksTag).IsEmpty()) {
return mediapipe::OkStatus();
}
const auto& image_size =
cc->Inputs().Tag(kImageSizeTag).Get<std::pair<int, int>>();
const auto& multi_face_landmarks =
cc->Inputs()
.Tag(kMultiFaceLandmarksTag)
.Get<std::vector<NormalizedLandmarkList>>();
auto multi_face_geometry =
absl::make_unique<std::vector<face_geometry::FaceGeometry>>();
ASSIGN_OR_RETURN(
*multi_face_geometry,
geometry_pipeline_->EstimateFaceGeometry(
multi_face_landmarks, //
/*frame_width*/ image_size.first,
/*frame_height*/ image_size.second),
_ << "Failed to estimate face geometry for multiple faces!");
cc->Outputs()
.Tag(kMultiFaceGeometryTag)
.AddPacket(mediapipe::Adopt<std::vector<face_geometry::FaceGeometry>>(
multi_face_geometry.release())
.At(cc->InputTimestamp()));
return mediapipe::OkStatus();
}
mediapipe::Status Close(CalculatorContext* cc) override {
return mediapipe::OkStatus();
}
private:
static mediapipe::StatusOr<face_geometry::GeometryPipelineMetadata>
ReadMetadataFromFile(const std::string& metadata_path) {
ASSIGN_OR_RETURN(std::string metadata_blob,
ReadContentBlobFromFile(metadata_path),
_ << "Failed to read a metadata blob from file!");
face_geometry::GeometryPipelineMetadata metadata;
RET_CHECK(metadata.ParseFromString(metadata_blob))
<< "Failed to parse a metadata proto from a binary blob!";
return metadata;
}
static mediapipe::StatusOr<std::string> ReadContentBlobFromFile(
const std::string& unresolved_path) {
ASSIGN_OR_RETURN(std::string resolved_path,
mediapipe::PathToResourceAsFile(unresolved_path),
_ << "Failed to resolve path! Path = " << unresolved_path);
std::string content_blob;
MP_RETURN_IF_ERROR(
mediapipe::GetResourceContents(resolved_path, &content_blob))
<< "Failed to read content blob! Resolved path = " << resolved_path;
return content_blob;
}
std::unique_ptr<face_geometry::GeometryPipeline> geometry_pipeline_;
};
} // namespace
using FaceGeometryPipelineCalculator = GeometryPipelineCalculator;
REGISTER_CALCULATOR(FaceGeometryPipelineCalculator);
} // namespace mediapipe
mediapipe/modules/face_geometry/geometry_pipeline_calculator.proto
+27
View File
@@ -0,0 +1,27 @@
// Copyright 2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto2";
package mediapipe;
import "mediapipe/framework/calculator_options.proto";
message FaceGeometryPipelineCalculatorOptions {
extend CalculatorOptions {
optional FaceGeometryPipelineCalculatorOptions ext = 323693812;
}
optional string metadata_path = 1;
}
mediapipe/modules/face_geometry/libs/BUILD
+103
View File
@@ -0,0 +1,103 @@
# Copyright 2020 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
licenses(["notice"])
package(default_visibility = ["//visibility:public"])
cc_library(
name = "effect_renderer",
srcs = ["effect_renderer.cc"],
hdrs = ["effect_renderer.h"],
deps = [
":mesh_3d_utils",
":validation_utils",
"//mediapipe/framework/formats:image_format_cc_proto",
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/formats:matrix_data_cc_proto",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"//mediapipe/framework/port:statusor",
"//mediapipe/gpu:gl_base",
"//mediapipe/gpu:shader_util",
"//mediapipe/modules/face_geometry/protos:environment_cc_proto",
"//mediapipe/modules/face_geometry/protos:face_geometry_cc_proto",
"//mediapipe/modules/face_geometry/protos:mesh_3d_cc_proto",
"@com_google_absl//absl/memory",
"@com_google_absl//absl/types:optional",
],
)
cc_library(
name = "geometry_pipeline",
srcs = ["geometry_pipeline.cc"],
hdrs = ["geometry_pipeline.h"],
deps = [
":mesh_3d_utils",
":procrustes_solver",
":validation_utils",
"//mediapipe/framework/formats:landmark_cc_proto",
"//mediapipe/framework/formats:matrix",
"//mediapipe/framework/formats:matrix_data_cc_proto",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"//mediapipe/framework/port:statusor",
"//mediapipe/modules/face_geometry/protos:environment_cc_proto",
"//mediapipe/modules/face_geometry/protos:face_geometry_cc_proto",
"//mediapipe/modules/face_geometry/protos:geometry_pipeline_metadata_cc_proto",
"//mediapipe/modules/face_geometry/protos:mesh_3d_cc_proto",
"@com_google_absl//absl/memory",
"@eigen_archive//:eigen",
],
)
cc_library(
name = "mesh_3d_utils",
srcs = ["mesh_3d_utils.cc"],
hdrs = ["mesh_3d_utils.h"],
deps = [
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:statusor",
"//mediapipe/modules/face_geometry/protos:mesh_3d_cc_proto",
],
)
cc_library(
name = "procrustes_solver",
srcs = ["procrustes_solver.cc"],
hdrs = ["procrustes_solver.h"],
deps = [
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"//mediapipe/framework/port:statusor",
"@com_google_absl//absl/memory",
"@eigen_archive//:eigen",
],
)
cc_library(
name = "validation_utils",
srcs = ["validation_utils.cc"],
hdrs = ["validation_utils.h"],
deps = [
":mesh_3d_utils",
"//mediapipe/framework/formats:matrix_data_cc_proto",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"//mediapipe/modules/face_geometry/protos:environment_cc_proto",
"//mediapipe/modules/face_geometry/protos:face_geometry_cc_proto",
"//mediapipe/modules/face_geometry/protos:geometry_pipeline_metadata_cc_proto",
"//mediapipe/modules/face_geometry/protos:mesh_3d_cc_proto",
],
)
mediapipe/modules/face_geometry/libs/effect_renderer.cc
+734
View File
@@ -0,0 +1,734 @@
// Copyright 2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/modules/face_geometry/libs/effect_renderer.h"
#include <cmath>
#include <cstdint>
#include <limits>
#include <memory>
#include <utility>
#include <vector>
#include "absl/memory/memory.h"
#include "absl/types/optional.h"
#include "mediapipe/framework/formats/image_format.pb.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/formats/matrix_data.pb.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/framework/port/status_macros.h"
#include "mediapipe/framework/port/statusor.h"
#include "mediapipe/gpu/gl_base.h"
#include "mediapipe/gpu/shader_util.h"
#include "mediapipe/modules/face_geometry/libs/mesh_3d_utils.h"
#include "mediapipe/modules/face_geometry/libs/validation_utils.h"
#include "mediapipe/modules/face_geometry/protos/environment.pb.h"
#include "mediapipe/modules/face_geometry/protos/face_geometry.pb.h"
#include "mediapipe/modules/face_geometry/protos/mesh_3d.pb.h"
namespace mediapipe::face_geometry {
namespace {
struct RenderableMesh3d {
static mediapipe::StatusOr<RenderableMesh3d> CreateFromProtoMesh3d(
const Mesh3d& proto_mesh_3d) {
Mesh3d::VertexType vertex_type = proto_mesh_3d.vertex_type();
RenderableMesh3d renderable_mesh_3d;
renderable_mesh_3d.vertex_size = GetVertexSize(vertex_type);
ASSIGN_OR_RETURN(
renderable_mesh_3d.vertex_position_size,
GetVertexComponentSize(vertex_type, VertexComponent::POSITION),
_ << "Failed to get the position vertex size!");
ASSIGN_OR_RETURN(
renderable_mesh_3d.tex_coord_position_size,
GetVertexComponentSize(vertex_type, VertexComponent::TEX_COORD),
_ << "Failed to get the tex coord vertex size!");
ASSIGN_OR_RETURN(
renderable_mesh_3d.vertex_position_offset,
GetVertexComponentOffset(vertex_type, VertexComponent::POSITION),
_ << "Failed to get the position vertex offset!");
ASSIGN_OR_RETURN(
renderable_mesh_3d.tex_coord_position_offset,
GetVertexComponentOffset(vertex_type, VertexComponent::TEX_COORD),
_ << "Failed to get the tex coord vertex offset!");
switch (proto_mesh_3d.primitive_type()) {
case Mesh3d::TRIANGLE:
renderable_mesh_3d.primitive_type = GL_TRIANGLES;
break;
default:
RET_CHECK_FAIL() << "Only triangle primitive types are supported!";
}
renderable_mesh_3d.vertex_buffer.reserve(
proto_mesh_3d.vertex_buffer_size());
for (float vertex_element : proto_mesh_3d.vertex_buffer()) {
renderable_mesh_3d.vertex_buffer.push_back(vertex_element);
}
renderable_mesh_3d.index_buffer.reserve(proto_mesh_3d.index_buffer_size());
for (uint32_t index_element : proto_mesh_3d.index_buffer()) {
RET_CHECK_LE(index_element, std::numeric_limits<uint16_t>::max())
<< "Index buffer elements must fit into the `uint16` type in order "
"to be renderable!";
renderable_mesh_3d.index_buffer.push_back(
static_cast<uint16_t>(index_element));
}
return renderable_mesh_3d;
}
uint32_t vertex_size;
uint32_t vertex_position_size;
uint32_t tex_coord_position_size;
uint32_t vertex_position_offset;
uint32_t tex_coord_position_offset;
uint32_t primitive_type;
std::vector<float> vertex_buffer;
std::vector<uint16_t> index_buffer;
};
class Texture {
public:
static mediapipe::StatusOr<std::unique_ptr<Texture>> WrapExternalTexture(
GLuint handle, GLenum target, int width, int height) {
RET_CHECK(handle) << "External texture must have a non-null handle!";
return absl::WrapUnique(new Texture(handle, target, width, height,
/*is_owned*/ false));
}
static mediapipe::StatusOr<std::unique_ptr<Texture>> CreateFromImageFrame(
const ImageFrame& image_frame) {
RET_CHECK(image_frame.IsAligned(ImageFrame::kGlDefaultAlignmentBoundary))
<< "Image frame memory must be aligned for GL usage!";
RET_CHECK(image_frame.Width() > 0 && image_frame.Height() > 0)
<< "Image frame must have positive dimensions!";
RET_CHECK(image_frame.Format() == ImageFormat::SRGB ||
image_frame.Format() == ImageFormat::SRGBA)
<< "Image frame format must be either SRGB or SRGBA!";
GLint image_format;
switch (image_frame.NumberOfChannels()) {
case 3:
image_format = GL_RGB;
break;
case 4:
image_format = GL_RGBA;
break;
default:
RET_CHECK_FAIL()
<< "Unexpected number of channels; expected 3 or 4, got "
<< image_frame.NumberOfChannels() << "!";
}
GLuint handle;
glGenTextures(1, &handle);
RET_CHECK(handle) << "Failed to initialize an OpenGL texture!";
glBindTexture(GL_TEXTURE_2D, handle);
glTexParameteri(GL_TEXTURE_2D, GL_NEAREST_MIPMAP_LINEAR, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
glTexImage2D(GL_TEXTURE_2D, 0, image_format, image_frame.Width(),
image_frame.Height(), 0, image_format, GL_UNSIGNED_BYTE,
image_frame.PixelData());
glGenerateMipmap(GL_TEXTURE_2D);
glBindTexture(GL_TEXTURE_2D, 0);
return absl::WrapUnique(new Texture(
handle, GL_TEXTURE_2D, image_frame.Width(), image_frame.Height(),
/*is_owned*/ true));
}
~Texture() {
if (is_owned_) {
glDeleteProgram(handle_);
}
}
GLuint handle() const { return handle_; }
GLenum target() const { return target_; }
int width() const { return width_; }
int height() const { return height_; }
private:
Texture(GLuint handle, GLenum target, int width, int height, bool is_owned)
: handle_(handle),
target_(target),
width_(width),
height_(height),
is_owned_(is_owned) {}
GLuint handle_;
GLenum target_;
int width_;
int height_;
bool is_owned_;
};
class RenderTarget {
public:
static mediapipe::StatusOr<std::unique_ptr<RenderTarget>> Create() {
GLuint framebuffer_handle;
glGenFramebuffers(1, &framebuffer_handle);
RET_CHECK(framebuffer_handle)
<< "Failed to initialize an OpenGL framebuffer!";
return absl::WrapUnique(new RenderTarget(framebuffer_handle));
}
~RenderTarget() {
glDeleteFramebuffers(1, &framebuffer_handle_);
// Renderbuffer handle might have never been created if this render target
// is destroyed before `SetColorbuffer()` is called for the first time.
if (renderbuffer_handle_) {
glDeleteFramebuffers(1, &renderbuffer_handle_);
}
}
mediapipe::Status SetColorbuffer(const Texture& colorbuffer_texture) {
glBindFramebuffer(GL_FRAMEBUFFER, framebuffer_handle_);
glViewport(0, 0, colorbuffer_texture.width(), colorbuffer_texture.height());
glActiveTexture(GL_TEXTURE0);
glBindTexture(colorbuffer_texture.target(), colorbuffer_texture.handle());
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0,
colorbuffer_texture.target(),
colorbuffer_texture.handle(),
/*level*/ 0);
glBindTexture(colorbuffer_texture.target(), 0);
// If the existing depth buffer has different dimensions, delete it.
if (renderbuffer_handle_ &&
(viewport_width_ != colorbuffer_texture.width() ||
viewport_height_ != colorbuffer_texture.height())) {
glDeleteRenderbuffers(1, &renderbuffer_handle_);
renderbuffer_handle_ = 0;
}
// If there is no depth buffer, create one.
if (!renderbuffer_handle_) {
glGenRenderbuffers(1, &renderbuffer_handle_);
RET_CHECK(renderbuffer_handle_)
<< "Failed to initialize an OpenGL renderbuffer!";
glBindRenderbuffer(GL_RENDERBUFFER, renderbuffer_handle_);
glRenderbufferStorage(GL_RENDERBUFFER, GL_DEPTH_COMPONENT16,
colorbuffer_texture.width(),
colorbuffer_texture.height());
glFramebufferRenderbuffer(GL_FRAMEBUFFER, GL_DEPTH_ATTACHMENT,
GL_RENDERBUFFER, renderbuffer_handle_);
glBindRenderbuffer(GL_RENDERBUFFER, 0);
}
viewport_width_ = colorbuffer_texture.width();
viewport_height_ = colorbuffer_texture.height();
glBindFramebuffer(GL_FRAMEBUFFER, 0);
glFlush();
return mediapipe::OkStatus();
}
void Bind() const {
glBindFramebuffer(GL_FRAMEBUFFER, framebuffer_handle_);
glViewport(0, 0, viewport_width_, viewport_height_);
}
void Unbind() const { glBindFramebuffer(GL_FRAMEBUFFER, 0); }
void Clear() const {
Bind();
glEnable(GL_DEPTH_TEST);
glDepthMask(GL_TRUE);
glClearColor(0.f, 0.f, 0.f, 0.f);
glClearDepthf(1.f);
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
glDepthMask(GL_FALSE);
glDisable(GL_DEPTH_TEST);
Unbind();
glFlush();
}
private:
explicit RenderTarget(GLuint framebuffer_handle)
: framebuffer_handle_(framebuffer_handle),
renderbuffer_handle_(0),
viewport_width_(-1),
viewport_height_(-1) {}
GLuint framebuffer_handle_;
GLuint renderbuffer_handle_;
int viewport_width_;
int viewport_height_;
};
class Renderer {
public:
enum class RenderMode { OPAQUE, OVERDRAW, OCCLUSION };
static mediapipe::StatusOr<std::unique_ptr<Renderer>> Create() {
static const GLint kAttrLocation[NUM_ATTRIBUTES] = {
ATTRIB_VERTEX,
ATTRIB_TEXTURE_POSITION,
};
static const GLchar* kAttrName[NUM_ATTRIBUTES] = {
"position",
"tex_coord",
};
static const GLchar* kVertSrc = R"(
uniform mat4 projection_mat;
uniform mat4 model_mat;
attribute vec4 position;
attribute vec4 tex_coord;
varying vec2 v_tex_coord;
void main() {
v_tex_coord = tex_coord.xy;
gl_Position = projection_mat * model_mat * position;
}
)";
static const GLchar* kFragSrc = R"(
precision mediump float;
varying vec2 v_tex_coord;
uniform sampler2D texture;
void main() {
gl_FragColor = texture2D(texture, v_tex_coord);
}
)";
GLuint program_handle = 0;
GlhCreateProgram(kVertSrc, kFragSrc, NUM_ATTRIBUTES,
(const GLchar**)&kAttrName[0], kAttrLocation,
&program_handle);
RET_CHECK(program_handle) << "Problem initializing the texture program!";
GLint projection_mat_uniform =
glGetUniformLocation(program_handle, "projection_mat");
GLint model_mat_uniform = glGetUniformLocation(program_handle, "model_mat");
GLint texture_uniform = glGetUniformLocation(program_handle, "texture");
RET_CHECK_NE(projection_mat_uniform, -1)
<< "Failed to find `projection_mat` uniform!";
RET_CHECK_NE(model_mat_uniform, -1)
<< "Failed to find `model_mat` uniform!";
RET_CHECK_NE(texture_uniform, -1) << "Failed to find `texture` uniform!";
return absl::WrapUnique(new Renderer(program_handle, projection_mat_uniform,
model_mat_uniform, texture_uniform));
}
~Renderer() { glDeleteProgram(program_handle_); }
mediapipe::Status Render(const RenderTarget& render_target,
const Texture& texture,
const RenderableMesh3d& mesh_3d,
const std::array<float, 16>& projection_mat,
const std::array<float, 16>& model_mat,
RenderMode render_mode) const {
glUseProgram(program_handle_);
// Set up the GL state.
glEnable(GL_BLEND);
glFrontFace(GL_CCW);
switch (render_mode) {
case RenderMode::OPAQUE:
glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);
glEnable(GL_DEPTH_TEST);
glDepthMask(GL_TRUE);
break;
case RenderMode::OVERDRAW:
glBlendFunc(GL_ONE, GL_ZERO);
glDisable(GL_DEPTH_TEST);
glDepthMask(GL_FALSE);
break;
case RenderMode::OCCLUSION:
glBlendFunc(GL_ZERO, GL_ONE);
glEnable(GL_DEPTH_TEST);
glDepthMask(GL_TRUE);
break;
}
render_target.Bind();
// Set up vertex attributes.
glVertexAttribPointer(
ATTRIB_VERTEX, mesh_3d.vertex_position_size, GL_FLOAT, 0,
mesh_3d.vertex_size * sizeof(float),
mesh_3d.vertex_buffer.data() + mesh_3d.vertex_position_offset);
glEnableVertexAttribArray(ATTRIB_VERTEX);
glVertexAttribPointer(
ATTRIB_TEXTURE_POSITION, mesh_3d.tex_coord_position_size, GL_FLOAT, 0,
mesh_3d.vertex_size * sizeof(float),
mesh_3d.vertex_buffer.data() + mesh_3d.tex_coord_position_offset);
glEnableVertexAttribArray(ATTRIB_TEXTURE_POSITION);
// Set up textures and uniforms.
glActiveTexture(GL_TEXTURE1);
glBindTexture(texture.target(), texture.handle());
glUniform1i(texture_uniform_, 1);
glUniformMatrix4fv(projection_mat_uniform_, 1, GL_FALSE,
projection_mat.data());
glUniformMatrix4fv(model_mat_uniform_, 1, GL_FALSE, model_mat.data());
// Draw the mesh.
glDrawElements(mesh_3d.primitive_type, mesh_3d.index_buffer.size(),
GL_UNSIGNED_SHORT, mesh_3d.index_buffer.data());
// Unbind textures and uniforms.
glActiveTexture(GL_TEXTURE1);
glBindTexture(texture.target(), 0);
render_target.Unbind();
// Unbind vertex attributes.
glDisableVertexAttribArray(ATTRIB_TEXTURE_POSITION);
glDisableVertexAttribArray(ATTRIB_VERTEX);
// Restore the GL state.
glDepthMask(GL_FALSE);
glDisable(GL_DEPTH_TEST);
glDisable(GL_BLEND);
glUseProgram(0);
glFlush();
return mediapipe::OkStatus();
}
private:
enum { ATTRIB_VERTEX, ATTRIB_TEXTURE_POSITION, NUM_ATTRIBUTES };
Renderer(GLuint program_handle, GLint projection_mat_uniform,
GLint model_mat_uniform, GLint texture_uniform)
: program_handle_(program_handle),
projection_mat_uniform_(projection_mat_uniform),
model_mat_uniform_(model_mat_uniform),
texture_uniform_(texture_uniform) {}
GLuint program_handle_;
GLint projection_mat_uniform_;
GLint model_mat_uniform_;
GLint texture_uniform_;
};
class EffectRendererImpl : public EffectRenderer {
public:
EffectRendererImpl(
const Environment& environment,
std::unique_ptr<RenderTarget> render_target,
std::unique_ptr<Renderer> renderer,
RenderableMesh3d&& renderable_quad_mesh_3d,
absl::optional<RenderableMesh3d>&& renderable_effect_mesh_3d,
std::unique_ptr<Texture> empty_color_texture,
std::unique_ptr<Texture> effect_texture)
: environment_(environment),
render_target_(std::move(render_target)),
renderer_(std::move(renderer)),
renderable_quad_mesh_3d_(std::move(renderable_quad_mesh_3d)),
renderable_effect_mesh_3d_(std::move(renderable_effect_mesh_3d)),
empty_color_texture_(std::move(empty_color_texture)),
effect_texture_(std::move(effect_texture)),
identity_matrix_(Create4x4IdentityMatrix()) {}
mediapipe::Status RenderEffect(
const std::vector<FaceGeometry>& multi_face_geometry,
int frame_width, //
int frame_height, //
GLenum src_texture_target, //
GLuint src_texture_name, //
GLenum dst_texture_target, //
GLuint dst_texture_name) {
// Validate input arguments.
MP_RETURN_IF_ERROR(ValidateFrameDimensions(frame_width, frame_height))
<< "Invalid frame dimensions!";
RET_CHECK(src_texture_name > 0 && dst_texture_name > 0)
<< "Both source and destination texture names must be non-null!";
RET_CHECK_NE(src_texture_name, dst_texture_name)
<< "Source and destination texture names must be different!";
// Validate all input face geometries.
for (const FaceGeometry& face_geometry : multi_face_geometry) {
MP_RETURN_IF_ERROR(ValidateFaceGeometry(face_geometry))
<< "Invalid face geometry!";
}
// Wrap both source and destination textures.
ASSIGN_OR_RETURN(
std::unique_ptr<Texture> src_texture,
Texture::WrapExternalTexture(src_texture_name, src_texture_target,
frame_width, frame_height),
_ << "Failed to wrap the external source texture");
ASSIGN_OR_RETURN(
std::unique_ptr<Texture> dst_texture,
Texture::WrapExternalTexture(dst_texture_name, dst_texture_target,
frame_width, frame_height),
_ << "Failed to wrap the external destination texture");
// Set the destination texture as the color buffer. Then, clear both the
// color and the depth buffers for the render target.
MP_RETURN_IF_ERROR(render_target_->SetColorbuffer(*dst_texture))
<< "Failed to set the destination texture as the colorbuffer!";
render_target_->Clear();
// Render the source texture on top of the quad mesh (i.e. make a copy)
// into the render target.
MP_RETURN_IF_ERROR(renderer_->Render(
*render_target_, *src_texture, renderable_quad_mesh_3d_,
identity_matrix_, identity_matrix_, Renderer::RenderMode::OVERDRAW))
<< "Failed to render the source texture on top of the quad mesh!";
// Extract pose transform matrices and meshes from the face geometry data;
const int num_faces = multi_face_geometry.size();
std::vector<std::array<float, 16>> face_pose_transform_matrices(num_faces);
std::vector<RenderableMesh3d> renderable_face_meshes(num_faces);
for (int i = 0; i < num_faces; ++i) {
const FaceGeometry& face_geometry = multi_face_geometry[i];
// Extract the face pose transformation matrix.
ASSIGN_OR_RETURN(
face_pose_transform_matrices[i],
Convert4x4MatrixDataToArrayFormat(
face_geometry.pose_transform_matrix()),
_ << "Failed to extract the face pose transformation matrix!");
// Extract the face mesh as a renderable.
ASSIGN_OR_RETURN(
renderable_face_meshes[i],
RenderableMesh3d::CreateFromProtoMesh3d(face_geometry.mesh()),
_ << "Failed to extract a renderable face mesh!");
}
// Create a perspective matrix using the frame aspect ratio.
std::array<float, 16> perspective_matrix = CreatePerspectiveMatrix(
/*aspect_ratio*/ static_cast<float>(frame_width) / frame_height);
// Render a face mesh occluder for each face.
for (int i = 0; i < num_faces; ++i) {
const std::array<float, 16>& face_pose_transform_matrix =
face_pose_transform_matrices[i];
const RenderableMesh3d& renderable_face_mesh = renderable_face_meshes[i];
// Render the face mesh using the empty color texture, i.e. the face
// mesh occluder.
//
// For occlusion, the pose transformation is moved ~1mm away from camera
// in order to allow the face mesh texture to be rendered without
// failing the depth test.
std::array<float, 16> occlusion_face_pose_transform_matrix =
face_pose_transform_matrix;
occlusion_face_pose_transform_matrix[14] -= 0.1f; // ~ 1mm
MP_RETURN_IF_ERROR(renderer_->Render(
*render_target_, *empty_color_texture_, renderable_face_mesh,
perspective_matrix, occlusion_face_pose_transform_matrix,
Renderer::RenderMode::OCCLUSION))
<< "Failed to render the face mesh occluder!";
}
// Render the main face mesh effect component for each face.
for (int i = 0; i < num_faces; ++i) {
const std::array<float, 16>& face_pose_transform_matrix =
face_pose_transform_matrices[i];
// If there is no effect 3D mesh provided, then the face mesh itself is
// used as a topology for rendering (for example, this can be used for
// facepaint effects or AR makeup).
const RenderableMesh3d& main_effect_mesh_3d =
renderable_effect_mesh_3d_ ? *renderable_effect_mesh_3d_
: renderable_face_meshes[i];
MP_RETURN_IF_ERROR(renderer_->Render(
*render_target_, *effect_texture_, main_effect_mesh_3d,
perspective_matrix, face_pose_transform_matrix,
Renderer::RenderMode::OPAQUE))
<< "Failed to render the main effect pass!";
}
// At this point in the code, the destination texture must contain the
// correctly renderer effect, so we should just return.
return mediapipe::OkStatus();
}
private:
std::array<float, 16> CreatePerspectiveMatrix(float aspect_ratio) const {
static constexpr float kDegreesToRadians = M_PI / 180.f;
std::array<float, 16> perspective_matrix;
perspective_matrix.fill(0.f);
const auto& env_camera = environment_.perspective_camera();
// Standard perspective projection matrix calculations.
const float f = 1.0f / std::tan(kDegreesToRadians *
env_camera.vertical_fov_degrees() / 2.f);
const float denom = 1.0f / (env_camera.near() - env_camera.far());
perspective_matrix[0] = f / aspect_ratio;
perspective_matrix[5] = f;
perspective_matrix[10] = (env_camera.near() + env_camera.far()) * denom;
perspective_matrix[11] = -1.f;
perspective_matrix[14] = 2.f * env_camera.far() * env_camera.near() * denom;
// If the environment's origin point location is in the top left corner,
// then skip additional flip along Y-axis is required to render correctly.
if (environment_.origin_point_location() ==
OriginPointLocation::TOP_LEFT_CORNER) {
perspective_matrix[5] *= -1.f;
}
return perspective_matrix;
}
static std::array<float, 16> Create4x4IdentityMatrix() {
return {1.f, 0.f, 0.f, 0.f, //
0.f, 1.f, 0.f, 0.f, //
0.f, 0.f, 1.f, 0.f, //
0.f, 0.f, 0.f, 1.f};
}
static mediapipe::StatusOr<std::array<float, 16>>
Convert4x4MatrixDataToArrayFormat(const MatrixData& matrix_data) {
RET_CHECK(matrix_data.rows() == 4 && //
matrix_data.cols() == 4 && //
matrix_data.packed_data_size() == 16)
<< "The matrix data must define a 4x4 matrix!";
std::array<float, 16> matrix_array;
for (int i = 0; i < 16; i++) {
matrix_array[i] = matrix_data.packed_data(i);
}
// Matrix array must be in the OpenGL-friendly column-major order. If
// `matrix_data` is in the row-major order, then transpose.
if (matrix_data.layout() == MatrixData::ROW_MAJOR) {
std::swap(matrix_array[1], matrix_array[4]);
std::swap(matrix_array[2], matrix_array[8]);
std::swap(matrix_array[3], matrix_array[12]);
std::swap(matrix_array[6], matrix_array[9]);
std::swap(matrix_array[7], matrix_array[13]);
std::swap(matrix_array[11], matrix_array[14]);
}
return matrix_array;
}
Environment environment_;
std::unique_ptr<RenderTarget> render_target_;
std::unique_ptr<Renderer> renderer_;
RenderableMesh3d renderable_quad_mesh_3d_;
absl::optional<RenderableMesh3d> renderable_effect_mesh_3d_;
std::unique_ptr<Texture> empty_color_texture_;
std::unique_ptr<Texture> effect_texture_;
std::array<float, 16> identity_matrix_;
};
Mesh3d CreateQuadMesh3d() {
static constexpr float kQuadMesh3dVertexBuffer[] = {
-1.f, -1.f, 0.f, 0.f, 0.f, //
1.f, -1.f, 0.f, 1.f, 0.f, //
-1.f, 1.f, 0.f, 0.f, 1.f, //
1.f, 1.f, 0.f, 1.f, 1.f, //
};
static constexpr uint16_t kQuadMesh3dIndexBuffer[] = {0, 1, 2, 1, 3, 2};
static constexpr int kQuadMesh3dVertexBufferSize =
sizeof(kQuadMesh3dVertexBuffer) / sizeof(float);
static constexpr int kQuadMesh3dIndexBufferSize =
sizeof(kQuadMesh3dIndexBuffer) / sizeof(uint16_t);
Mesh3d quad_mesh_3d;
quad_mesh_3d.set_vertex_type(Mesh3d::VERTEX_PT);
quad_mesh_3d.set_primitive_type(Mesh3d::TRIANGLE);
for (int i = 0; i < kQuadMesh3dVertexBufferSize; ++i) {
quad_mesh_3d.add_vertex_buffer(kQuadMesh3dVertexBuffer[i]);
}
for (int i = 0; i < kQuadMesh3dIndexBufferSize; ++i) {
quad_mesh_3d.add_index_buffer(kQuadMesh3dIndexBuffer[i]);
}
return quad_mesh_3d;
}
ImageFrame CreateEmptyColorTexture() {
static constexpr ImageFormat::Format kEmptyColorTextureFormat =
ImageFormat::SRGBA;
static constexpr int kEmptyColorTextureWidth = 1;
static constexpr int kEmptyColorTextureHeight = 1;
ImageFrame empty_color_texture(
kEmptyColorTextureFormat, kEmptyColorTextureWidth,
kEmptyColorTextureHeight, ImageFrame::kGlDefaultAlignmentBoundary);
empty_color_texture.SetToZero();
return empty_color_texture;
}
} // namespace
mediapipe::StatusOr<std::unique_ptr<EffectRenderer>> CreateEffectRenderer(
const Environment& environment, //
const absl::optional<Mesh3d>& effect_mesh_3d, //
ImageFrame&& effect_texture) {
MP_RETURN_IF_ERROR(ValidateEnvironment(environment))
<< "Invalid environment!";
if (effect_mesh_3d) {
MP_RETURN_IF_ERROR(ValidateMesh3d(*effect_mesh_3d))
<< "Invalid effect 3D mesh!";
}
ASSIGN_OR_RETURN(std::unique_ptr<RenderTarget> render_target,
RenderTarget::Create(),
_ << "Failed to create a render target!");
ASSIGN_OR_RETURN(std::unique_ptr<Renderer> renderer, Renderer::Create(),
_ << "Failed to create a renderer!");
ASSIGN_OR_RETURN(RenderableMesh3d renderable_quad_mesh_3d,
RenderableMesh3d::CreateFromProtoMesh3d(CreateQuadMesh3d()),
_ << "Failed to create a renderable quad mesh!");
absl::optional<RenderableMesh3d> renderable_effect_mesh_3d;
if (effect_mesh_3d) {
ASSIGN_OR_RETURN(renderable_effect_mesh_3d,
RenderableMesh3d::CreateFromProtoMesh3d(*effect_mesh_3d),
_ << "Failed to create a renderable effect mesh!");
}
ASSIGN_OR_RETURN(std::unique_ptr<Texture> empty_color_gl_texture,
Texture::CreateFromImageFrame(CreateEmptyColorTexture()),
_ << "Failed to create an empty color texture!");
ASSIGN_OR_RETURN(std::unique_ptr<Texture> effect_gl_texture,
Texture::CreateFromImageFrame(effect_texture),
_ << "Failed to create an effect texture!");
std::unique_ptr<EffectRenderer> result =
absl::make_unique<EffectRendererImpl>(
environment, std::move(render_target), std::move(renderer),
std::move(renderable_quad_mesh_3d),
std::move(renderable_effect_mesh_3d),
std::move(empty_color_gl_texture), std::move(effect_gl_texture));
return result;
}
} // namespace mediapipe::face_geometry
mediapipe/modules/face_geometry/libs/effect_renderer.h
+92
View File
@@ -0,0 +1,92 @@
// Copyright 2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef MEDIAPIPE_MODULES_FACE_GEOMETRY_LIBS_EFFECT_RENDERER_H_
#define MEDIAPIPE_MODULES_FACE_GEOMETRY_LIBS_EFFECT_RENDERER_H_
#include <memory>
#include <vector>
#include "absl/types/optional.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/framework/port/statusor.h"
#include "mediapipe/gpu/gl_base.h"
#include "mediapipe/modules/face_geometry/protos/environment.pb.h"
#include "mediapipe/modules/face_geometry/protos/face_geometry.pb.h"
#include "mediapipe/modules/face_geometry/protos/mesh_3d.pb.h"
namespace mediapipe::face_geometry {
// Encapsulates a stateful face effect renderer.
class EffectRenderer {
public:
virtual ~EffectRenderer() = default;
// Renders a face effect based on the multiple facial geometries.
//
// Must be called in the same GL context as was used upon initialization.
//
// Each of the `multi_face_geometry` must be valid (for details, please refer
// to the proto message definition comments and/or `validation_utils.h/cc`).
// Additionally, all face mesh index buffer elements must fit into the
// `uint16` type in order to be renderable.
//
// Both `frame_width` and `frame_height` must be positive.
//
// Both `src_texture_name` and `dst_texture_name` must be positive and
// reference existing OpenGL textures in the current context. They should also
// reference different textures as the in-place effect rendering is not yet
// supported.
virtual mediapipe::Status RenderEffect(
const std::vector<FaceGeometry>& multi_face_geometry,
int frame_width, //
int frame_height, //
GLenum src_texture_target, //
GLuint src_texture_name, //
GLenum dst_texture_target, //
GLuint dst_texture_name) = 0;
};
// Creates an instance of `EffectRenderer`.
//
// `effect_mesh_3d` defines a rigid 3d mesh which is "attached" to the face and
// is driven by the face pose transformation matrix. If is not present, the
// runtime face mesh will be used as the effect mesh - this mode is handy for
// facepaint effects. In both rendering modes, the face mesh is first rendered
// as an occluder straight into the depth buffer. This step helps to create a
// more believable effect via hiding invisible elements behind the face surface.
//
// `effect_texture` defines the color texture to be rendered on top of the
// effect mesh. Please be aware about the difference between the CPU texture
// memory layout and the GPU texture sampler coordinate space. This renderer
// follows conventions discussed here: https://open.gl/textures
//
// Must be called in the same GL context as will be used for rendering.
//
// Both `environment` and `effect_mesh_3d` (is present) must be valid (for
// details, please refer to the proto message definition comments and/or
// `validation_utils.h/cc`). Additionally, `effect_mesh_3d`s index buffer
// elements must fit into the `uint16` type in order to be renderable.
//
// `effect_texture` must have positive dimensions. Its format must be either
// `SRGB` or `SRGBA`. Its memory must be aligned for GL usage.
mediapipe::StatusOr<std::unique_ptr<EffectRenderer>> CreateEffectRenderer(
const Environment& environment, //
const absl::optional<Mesh3d>& effect_mesh_3d, //
ImageFrame&& effect_texture);
} // namespace mediapipe::face_geometry
#endif // MEDIAPIPE_MODULES_FACE_GEOMETRY_LIBS_EFFECT_RENDERER_H_
mediapipe/modules/face_geometry/libs/geometry_pipeline.cc
+434
View File
@@ -0,0 +1,434 @@
// Copyright 2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/modules/face_geometry/libs/geometry_pipeline.h"
#include <cmath>
#include <cstdint>
#include <memory>
#include <utility>
#include <vector>
#include "Eigen/Core"
#include "absl/memory/memory.h"
#include "mediapipe/framework/formats/landmark.pb.h"
#include "mediapipe/framework/formats/matrix.h"
#include "mediapipe/framework/formats/matrix_data.pb.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/framework/port/status_macros.h"
#include "mediapipe/framework/port/statusor.h"
#include "mediapipe/modules/face_geometry/libs/mesh_3d_utils.h"
#include "mediapipe/modules/face_geometry/libs/procrustes_solver.h"
#include "mediapipe/modules/face_geometry/libs/validation_utils.h"
#include "mediapipe/modules/face_geometry/protos/environment.pb.h"
#include "mediapipe/modules/face_geometry/protos/face_geometry.pb.h"
#include "mediapipe/modules/face_geometry/protos/geometry_pipeline_metadata.pb.h"
#include "mediapipe/modules/face_geometry/protos/mesh_3d.pb.h"
namespace mediapipe::face_geometry {
namespace {
struct PerspectiveCameraFrustum {
// NOTE: all arguments must be validated prior to calling this constructor.
PerspectiveCameraFrustum(const PerspectiveCamera& perspective_camera,
int frame_width, int frame_height) {
static constexpr float kDegreesToRadians = 3.14159265358979323846f / 180.f;
const float height_at_near =
2.f * perspective_camera.near() *
std::tan(0.5f * kDegreesToRadians *
perspective_camera.vertical_fov_degrees());
const float width_at_near = frame_width * height_at_near / frame_height;
left = -0.5f * width_at_near;
right = 0.5f * width_at_near;
bottom = -0.5f * height_at_near;
top = 0.5f * height_at_near;
near = perspective_camera.near();
far = perspective_camera.far();
}
float left;
float right;
float bottom;
float top;
float near;
float far;
};
class ScreenToMetricSpaceConverter {
public:
ScreenToMetricSpaceConverter(
OriginPointLocation origin_point_location, //
Eigen::Matrix3Xf&& canonical_metric_landmarks, //
Eigen::VectorXf&& landmark_weights, //
std::unique_ptr<ProcrustesSolver> procrustes_solver)
: origin_point_location_(origin_point_location),
canonical_metric_landmarks_(std::move(canonical_metric_landmarks)),
landmark_weights_(std::move(landmark_weights)),
procrustes_solver_(std::move(procrustes_solver)) {}
// Converts `screen_landmark_list` into `metric_landmark_list` and estimates
// the `pose_transform_mat`.
//
// Here's the algorithm summary:
//
// (1) Project X- and Y- screen landmark coordinates at the Z near plane.
//
// (2) Estimate a canonical-to-runtime landmark set scale by running the
// Procrustes solver using the screen runtime landmarks.
//
// On this iteration, screen landmarks are used instead of unprojected
// metric landmarks as it is not safe to unproject due to the relative
// nature of the input screen landmark Z coordinate.
//
// (3) Use the canonical-to-runtime scale from (2) to unproject the screen
// landmarks. The result is referenced as "intermediate landmarks" because
// they are the first estimation of the resuling metric landmarks, but are
// not quite there yet.
//
// (4) Estimate a canonical-to-runtime landmark set scale by running the
// Procrustes solver using the intermediate runtime landmarks.
//
// (5) Use the product of the scale factors from (2) and (4) to unproject
// the screen landmarks the second time. This is the second and the final
// estimation of the metric landmarks.
//
// (6) Multiply each of the metric landmarks by the inverse pose
// transformation matrix to align the runtime metric face landmarks with
// the canonical metric face landmarks.
//
// Note: the input screen landmarks are in the left-handed coordinate system,
// however any metric landmarks - including the canonical metric
// landmarks, the final runtime metric landmarks and any intermediate
// runtime metric landmarks - are in the right-handed coordinate system.
//
// To keep the logic correct, the landmark set handedness is changed any
// time the screen-to-metric semantic barrier is passed.
mediapipe::Status Convert(
const NormalizedLandmarkList& screen_landmark_list, //
const PerspectiveCameraFrustum& pcf, //
LandmarkList& metric_landmark_list, //
Eigen::Matrix4f& pose_transform_mat) const {
RET_CHECK_EQ(screen_landmark_list.landmark_size(),
canonical_metric_landmarks_.cols())
<< "The number of landmarks doesn't match the number passed upon "
"initialization!";
Eigen::Matrix3Xf screen_landmarks;
ConvertLandmarkListToEigenMatrix(screen_landmark_list, screen_landmarks);
ProjectXY(pcf, screen_landmarks);
const float depth_offset = screen_landmarks.row(2).mean();
// 1st iteration: don't unproject XY because it's unsafe to do so due to
// the relative nature of the Z coordinate. Instead, run the
// first estimation on the projected XY and use that scale to
// unproject for the 2nd iteration.
Eigen::Matrix3Xf intermediate_landmarks(screen_landmarks);
ChangeHandedness(intermediate_landmarks);
ASSIGN_OR_RETURN(const float first_iteration_scale,
EstimateScale(intermediate_landmarks),
_ << "Failed to estimate first iteration scale!");
// 2nd iteration: unproject XY using the scale from the 1st iteration.
intermediate_landmarks = screen_landmarks;
MoveAndRescaleZ(pcf, depth_offset, first_iteration_scale,
intermediate_landmarks);
UnprojectXY(pcf, intermediate_landmarks);
ChangeHandedness(intermediate_landmarks);
ASSIGN_OR_RETURN(const float second_iteration_scale,
EstimateScale(intermediate_landmarks),
_ << "Failed to estimate second iteration scale!");
// Use the total scale to unproject the screen landmarks.
float total_scale = first_iteration_scale * second_iteration_scale;
MoveAndRescaleZ(pcf, depth_offset, total_scale, screen_landmarks);
UnprojectXY(pcf, screen_landmarks);
ChangeHandedness(screen_landmarks);
// At this point, screen landmarks are converted into metric landmarks.
Eigen::Matrix3Xf& metric_landmarks = screen_landmarks;
MP_RETURN_IF_ERROR(procrustes_solver_->SolveWeightedOrthogonalProblem(
canonical_metric_landmarks_, metric_landmarks, landmark_weights_,
pose_transform_mat))
<< "Failed to estimate pose transform matrix!";
// Multiply each of the metric landmarks by the inverse pose transformation
// matrix to align the runtime metric face landmarks with the canonical
// metric face landmarks.
Eigen::Matrix4f inv_pose_transform_mat = pose_transform_mat.inverse();
auto inv_pose_rotation = inv_pose_transform_mat.leftCols(3).topRows(3);
auto inv_pose_translation = inv_pose_transform_mat.col(3).topRows(3);
metric_landmarks =
(inv_pose_rotation * metric_landmarks).colwise() + inv_pose_translation;
ConvertEigenMatrixToLandmarkList(metric_landmarks, metric_landmark_list);
return mediapipe::OkStatus();
}
private:
void ProjectXY(const PerspectiveCameraFrustum& pcf,
Eigen::Matrix3Xf& landmarks) const {
float x_scale = pcf.right - pcf.left;
float y_scale = pcf.top - pcf.bottom;
float x_translation = pcf.left;
float y_translation = pcf.bottom;
if (origin_point_location_ == OriginPointLocation::TOP_LEFT_CORNER) {
landmarks.row(1) = 1.f - landmarks.row(1).array();
}
landmarks =
landmarks.array().colwise() * Eigen::Array3f(x_scale, y_scale, x_scale);
landmarks.colwise() += Eigen::Vector3f(x_translation, y_translation, 0.f);
}
mediapipe::StatusOr<float> EstimateScale(Eigen::Matrix3Xf& landmarks) const {
Eigen::Matrix4f transform_mat;
MP_RETURN_IF_ERROR(procrustes_solver_->SolveWeightedOrthogonalProblem(
canonical_metric_landmarks_, landmarks, landmark_weights_,
transform_mat))
<< "Failed to estimate canonical-to-runtime landmark set transform!";
return transform_mat.col(0).norm();
}
static void MoveAndRescaleZ(const PerspectiveCameraFrustum& pcf,
float depth_offset, float scale,
Eigen::Matrix3Xf& landmarks) {
landmarks.row(2) =
(landmarks.array().row(2) - depth_offset + pcf.near) / scale;
}
static void UnprojectXY(const PerspectiveCameraFrustum& pcf,
Eigen::Matrix3Xf& landmarks) {
landmarks.row(0) =
landmarks.row(0).cwiseProduct(landmarks.row(2)) / pcf.near;
landmarks.row(1) =
landmarks.row(1).cwiseProduct(landmarks.row(2)) / pcf.near;
}
static void ChangeHandedness(Eigen::Matrix3Xf& landmarks) {
landmarks.row(2) *= -1.f;
}
static void ConvertLandmarkListToEigenMatrix(
const NormalizedLandmarkList& landmark_list,
Eigen::Matrix3Xf& eigen_matrix) {
eigen_matrix = Eigen::Matrix3Xf(3, landmark_list.landmark_size());
for (int i = 0; i < landmark_list.landmark_size(); ++i) {
const auto& landmark = landmark_list.landmark(i);
eigen_matrix(0, i) = landmark.x();
eigen_matrix(1, i) = landmark.y();
eigen_matrix(2, i) = landmark.z();
}
}
static void ConvertEigenMatrixToLandmarkList(
const Eigen::Matrix3Xf& eigen_matrix, LandmarkList& landmark_list) {
landmark_list.Clear();
for (int i = 0; i < eigen_matrix.cols(); ++i) {
auto& landmark = *landmark_list.add_landmark();
landmark.set_x(eigen_matrix(0, i));
landmark.set_y(eigen_matrix(1, i));
landmark.set_z(eigen_matrix(2, i));
}
}
OriginPointLocation origin_point_location_;
Eigen::Matrix3Xf canonical_metric_landmarks_;
Eigen::VectorXf landmark_weights_;
std::unique_ptr<ProcrustesSolver> procrustes_solver_;
};
class GeometryPipelineImpl : public GeometryPipeline {
public:
GeometryPipelineImpl(
const PerspectiveCamera& perspective_camera, //
const Mesh3d& canonical_mesh, //
uint32_t canonical_mesh_vertex_size, //
uint32_t canonical_mesh_num_vertices,
uint32_t canonical_mesh_vertex_position_offset,
std::unique_ptr<ScreenToMetricSpaceConverter> space_converter)
: perspective_camera_(perspective_camera),
canonical_mesh_(canonical_mesh),
canonical_mesh_vertex_size_(canonical_mesh_vertex_size),
canonical_mesh_num_vertices_(canonical_mesh_num_vertices),
canonical_mesh_vertex_position_offset_(
canonical_mesh_vertex_position_offset),
space_converter_(std::move(space_converter)) {}
mediapipe::StatusOr<std::vector<FaceGeometry>> EstimateFaceGeometry(
const std::vector<NormalizedLandmarkList>& multi_face_landmarks,
int frame_width, int frame_height) const override {
MP_RETURN_IF_ERROR(ValidateFrameDimensions(frame_width, frame_height))
<< "Invalid frame dimensions!";
// Create a perspective camera frustum to be shared for geometry estimation
// per each face.
PerspectiveCameraFrustum pcf(perspective_camera_, frame_width,
frame_height);
std::vector<FaceGeometry> multi_face_geometry;
// From this point, the meaning of "face landmarks" is clarified further as
// "screen face landmarks". This is done do distinguish from "metric face
// landmarks" that are derived during the face geometry estimation process.
for (const NormalizedLandmarkList& screen_face_landmarks :
multi_face_landmarks) {
// Having a too compact screen landmark list will result in numerical
// instabilities, therefore such faces are filtered.
if (IsScreenLandmarkListTooCompact(screen_face_landmarks)) {
continue;
}
// Convert the screen landmarks into the metric landmarks and
// get the pose transformation matrix.
LandmarkList metric_face_landmarks;
Eigen::Matrix4f pose_transform_mat;
MP_RETURN_IF_ERROR(space_converter_->Convert(screen_face_landmarks, pcf,
metric_face_landmarks,
pose_transform_mat))
<< "Failed to convert landmarks from the screen to the metric space!";
// Pack geometry data for this face.
FaceGeometry face_geometry;
Mesh3d* mutable_mesh = face_geometry.mutable_mesh();
// Copy the canonical face mesh as the face geometry mesh.
mutable_mesh->CopyFrom(canonical_mesh_);
// Replace XYZ vertex mesh coodinates with the metric landmark positions.
for (int i = 0; i < canonical_mesh_num_vertices_; ++i) {
uint32_t vertex_buffer_offset = canonical_mesh_vertex_size_ * i +
canonical_mesh_vertex_position_offset_;
mutable_mesh->set_vertex_buffer(vertex_buffer_offset,
metric_face_landmarks.landmark(i).x());
mutable_mesh->set_vertex_buffer(vertex_buffer_offset + 1,
metric_face_landmarks.landmark(i).y());
mutable_mesh->set_vertex_buffer(vertex_buffer_offset + 2,
metric_face_landmarks.landmark(i).z());
}
// Populate the face pose transformation matrix.
mediapipe::MatrixDataProtoFromMatrix(
pose_transform_mat, face_geometry.mutable_pose_transform_matrix());
multi_face_geometry.push_back(face_geometry);
}
return multi_face_geometry;
}
private:
static bool IsScreenLandmarkListTooCompact(
const NormalizedLandmarkList& screen_landmarks) {
float mean_x = 0.f;
float mean_y = 0.f;
for (int i = 0; i < screen_landmarks.landmark_size(); ++i) {
const auto& landmark = screen_landmarks.landmark(i);
mean_x += (landmark.x() - mean_x) / static_cast<float>(i + 1);
mean_y += (landmark.y() - mean_y) / static_cast<float>(i + 1);
}
float max_sq_dist = 0.f;
for (const auto& landmark : screen_landmarks.landmark()) {
const float d_x = landmark.x() - mean_x;
const float d_y = landmark.y() - mean_y;
max_sq_dist = std::max(max_sq_dist, d_x * d_x + d_y * d_y);
}
static constexpr float kIsScreenLandmarkListTooCompactThreshold = 1e-3f;
return std::sqrt(max_sq_dist) <= kIsScreenLandmarkListTooCompactThreshold;
}
const PerspectiveCamera perspective_camera_;
const Mesh3d canonical_mesh_;
const uint32_t canonical_mesh_vertex_size_;
const uint32_t canonical_mesh_num_vertices_;
const uint32_t canonical_mesh_vertex_position_offset_;
std::unique_ptr<ScreenToMetricSpaceConverter> space_converter_;
};
} // namespace
mediapipe::StatusOr<std::unique_ptr<GeometryPipeline>> CreateGeometryPipeline(
const Environment& environment, const GeometryPipelineMetadata& metadata) {
MP_RETURN_IF_ERROR(ValidateEnvironment(environment))
<< "Invalid environment!";
MP_RETURN_IF_ERROR(ValidateGeometryPipelineMetadata(metadata))
<< "Invalid geometry pipeline metadata!";
const auto& canonical_mesh = metadata.canonical_mesh();
RET_CHECK(HasVertexComponent(canonical_mesh.vertex_type(),
VertexComponent::POSITION))
<< "Canonical face mesh must have the `POSITION` vertex component!";
RET_CHECK(HasVertexComponent(canonical_mesh.vertex_type(),
VertexComponent::TEX_COORD))
<< "Canonical face mesh must have the `TEX_COORD` vertex component!";
uint32_t canonical_mesh_vertex_size =
GetVertexSize(canonical_mesh.vertex_type());
uint32_t canonical_mesh_num_vertices =
canonical_mesh.vertex_buffer_size() / canonical_mesh_vertex_size;
uint32_t canonical_mesh_vertex_position_offset =
GetVertexComponentOffset(canonical_mesh.vertex_type(),
VertexComponent::POSITION)
.ValueOrDie();
// Put the Procrustes landmark basis into Eigen matrices for an easier access.
Eigen::Matrix3Xf canonical_metric_landmarks =
Eigen::Matrix3Xf::Zero(3, canonical_mesh_num_vertices);
Eigen::VectorXf landmark_weights =
Eigen::VectorXf::Zero(canonical_mesh_num_vertices);
for (int i = 0; i < canonical_mesh_num_vertices; ++i) {
uint32_t vertex_buffer_offset =
canonical_mesh_vertex_size * i + canonical_mesh_vertex_position_offset;
canonical_metric_landmarks(0, i) =
canonical_mesh.vertex_buffer(vertex_buffer_offset);
canonical_metric_landmarks(1, i) =
canonical_mesh.vertex_buffer(vertex_buffer_offset + 1);
canonical_metric_landmarks(2, i) =
canonical_mesh.vertex_buffer(vertex_buffer_offset + 2);
}
for (const WeightedLandmarkRef& wlr : metadata.procrustes_landmark_basis()) {
uint32_t landmark_id = wlr.landmark_id();
landmark_weights(landmark_id) = wlr.weight();
}
std::unique_ptr<GeometryPipeline> result =
absl::make_unique<GeometryPipelineImpl>(
environment.perspective_camera(), canonical_mesh,
canonical_mesh_vertex_size, canonical_mesh_num_vertices,
canonical_mesh_vertex_position_offset,
absl::make_unique<ScreenToMetricSpaceConverter>(
environment.origin_point_location(),
std::move(canonical_metric_landmarks),
std::move(landmark_weights),
CreateFloatPrecisionProcrustesSolver()));
return result;
}
} // namespace mediapipe::face_geometry
mediapipe/modules/face_geometry/libs/geometry_pipeline.h
+67
View File
@@ -0,0 +1,67 @@
// Copyright 2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef MEDIAPIPE_FACE_GEOMETRY_LIBS_GEOMETRY_PIPELINE_H_
#define MEDIAPIPE_FACE_GEOMETRY_LIBS_GEOMETRY_PIPELINE_H_
#include <memory>
#include <vector>
#include "mediapipe/framework/formats/landmark.pb.h"
#include "mediapipe/framework/port/statusor.h"
#include "mediapipe/modules/face_geometry/protos/environment.pb.h"
#include "mediapipe/modules/face_geometry/protos/face_geometry.pb.h"
#include "mediapipe/modules/face_geometry/protos/geometry_pipeline_metadata.pb.h"
namespace mediapipe::face_geometry {
// Encapsulates a stateless estimator of facial geometry in a Metric space based
// on the normalized face landmarks in the Screen space.
class GeometryPipeline {
public:
virtual ~GeometryPipeline() = default;
// Estimates geometry data for multiple faces.
//
// Returns an error status if any of the passed arguments is invalid.
//
// The result includes face geometry data for a subset of the input faces,
// however geometry data for some faces might be missing. This may happen if
// it'd be unstable to estimate the facial geometry based on a corresponding
// face landmark list for any reason (for example, if the landmark list is too
// compact).
//
// Each face landmark list must have the same number of landmarks as was
// passed upon initialization via the canonical face mesh (as a part of the
// geometry pipeline metadata).
//
// Both `frame_width` and `frame_height` must be positive.
virtual mediapipe::StatusOr<std::vector<FaceGeometry>> EstimateFaceGeometry(
const std::vector<NormalizedLandmarkList>& multi_face_landmarks,
int frame_width, int frame_height) const = 0;
};
// Creates an instance of `GeometryPipeline`.
//
// Both `environment` and `metadata` must be valid (for details, please refer to
// the proto message definition comments and/or `validation_utils.h/cc`).
//
// Canonical face mesh (defined as a part of `metadata`) must have the
// `POSITION` and the `TEX_COORD` vertex components.
mediapipe::StatusOr<std::unique_ptr<GeometryPipeline>> CreateGeometryPipeline(
const Environment& environment, const GeometryPipelineMetadata& metadata);
} // namespace mediapipe::face_geometry
#endif // MEDIAPIPE_FACE_GEOMETRY_LIBS_GEOMETRY_PIPELINE_H_
mediapipe/modules/face_geometry/libs/mesh_3d_utils.cc
+103
View File
@@ -0,0 +1,103 @@
// Copyright 2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/modules/face_geometry/libs/mesh_3d_utils.h"
#include <cstdint>
#include <cstdlib>
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/statusor.h"
#include "mediapipe/modules/face_geometry/protos/mesh_3d.pb.h"
namespace mediapipe::face_geometry {
namespace {
bool HasVertexComponentVertexPT(VertexComponent vertex_component) {
switch (vertex_component) {
case VertexComponent::POSITION:
case VertexComponent::TEX_COORD:
return true;
default:
return false;
}
}
uint32_t GetVertexComponentSizeVertexPT(VertexComponent vertex_component) {
switch (vertex_component) {
case VertexComponent::POSITION:
return 3;
case VertexComponent::TEX_COORD:
return 2;
}
}
uint32_t GetVertexComponentOffsetVertexPT(VertexComponent vertex_component) {
switch (vertex_component) {
case VertexComponent::POSITION:
return 0;
case VertexComponent::TEX_COORD:
return GetVertexComponentSizeVertexPT(VertexComponent::POSITION);
}
}
} // namespace
std::size_t GetVertexSize(Mesh3d::VertexType vertex_type) {
switch (vertex_type) {
case Mesh3d::VERTEX_PT:
return GetVertexComponentSizeVertexPT(VertexComponent::POSITION) +
GetVertexComponentSizeVertexPT(VertexComponent::TEX_COORD);
}
}
std::size_t GetPrimitiveSize(Mesh3d::PrimitiveType primitive_type) {
switch (primitive_type) {
case Mesh3d::TRIANGLE:
return 3;
}
}
bool HasVertexComponent(Mesh3d::VertexType vertex_type,
VertexComponent vertex_component) {
switch (vertex_type) {
case Mesh3d::VERTEX_PT:
return HasVertexComponentVertexPT(vertex_component);
}
}
mediapipe::StatusOr<uint32_t> GetVertexComponentOffset(
Mesh3d::VertexType vertex_type, VertexComponent vertex_component) {
RET_CHECK(HasVertexComponentVertexPT(vertex_component))
<< "A given vertex type doesn't have the requested component!";
switch (vertex_type) {
case Mesh3d::VERTEX_PT:
return GetVertexComponentOffsetVertexPT(vertex_component);
}
}
mediapipe::StatusOr<uint32_t> GetVertexComponentSize(
Mesh3d::VertexType vertex_type, VertexComponent vertex_component) {
RET_CHECK(HasVertexComponentVertexPT(vertex_component))
<< "A given vertex type doesn't have the requested component!";
switch (vertex_type) {
case Mesh3d::VERTEX_PT:
return GetVertexComponentSizeVertexPT(vertex_component);
}
}
} // namespace mediapipe::face_geometry
mediapipe/modules/face_geometry/libs/mesh_3d_utils.h
+51
View File
@@ -0,0 +1,51 @@
// Copyright 2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef MEDIAPIPE_FACE_GEOMETRY_LIBS_MESH_3D_UTILS_H_
#define MEDIAPIPE_FACE_GEOMETRY_LIBS_MESH_3D_UTILS_H_
#include <cstdint>
#include <cstdlib>
#include "mediapipe/framework/port/statusor.h"
#include "mediapipe/modules/face_geometry/protos/mesh_3d.pb.h"
namespace mediapipe::face_geometry {
enum class VertexComponent { POSITION, TEX_COORD };
std::size_t GetVertexSize(Mesh3d::VertexType vertex_type);
std::size_t GetPrimitiveSize(Mesh3d::PrimitiveType primitive_type);
bool HasVertexComponent(Mesh3d::VertexType vertex_type,
VertexComponent vertex_component);
// Computes the vertex component offset.
//
// Returns an error status if a given vertex type doesn't have the requested
// component.
mediapipe::StatusOr<uint32_t> GetVertexComponentOffset(
Mesh3d::VertexType vertex_type, VertexComponent vertex_component);
// Computes the vertex component size.
//
// Returns an error status if a given vertex type doesn't have the requested
// component.
mediapipe::StatusOr<uint32_t> GetVertexComponentSize(
Mesh3d::VertexType vertex_type, VertexComponent vertex_component);
} // namespace mediapipe::face_geometry
#endif // MEDIAPIPE_FACE_GEOMETRY_LIBS_MESH_3D_UTILS_H_
mediapipe/modules/face_geometry/libs/procrustes_solver.cc
+266
View File
@@ -0,0 +1,266 @@
// Copyright 2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/modules/face_geometry/libs/procrustes_solver.h"
#include <cmath>
#include <memory>
#include "Eigen/Dense"
#include "absl/memory/memory.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/framework/port/status_macros.h"
#include "mediapipe/framework/port/statusor.h"
namespace mediapipe {
namespace face_geometry {
namespace {
class FloatPrecisionProcrustesSolver : public ProcrustesSolver {
public:
FloatPrecisionProcrustesSolver() = default;
mediapipe::Status SolveWeightedOrthogonalProblem(
const Eigen::Matrix3Xf& source_points, //
const Eigen::Matrix3Xf& target_points, //
const Eigen::VectorXf& point_weights,
Eigen::Matrix4f& transform_mat) const override {
// Validate inputs.
MP_RETURN_IF_ERROR(ValidateInputPoints(source_points, target_points))
<< "Failed to validate weighted orthogonal problem input points!";
MP_RETURN_IF_ERROR(
ValidatePointWeights(source_points.cols(), point_weights))
<< "Failed to validate weighted orthogonal problem point weights!";
// Extract square root from the point weights.
Eigen::VectorXf sqrt_weights = ExtractSquareRoot(point_weights);
// Try to solve the WEOP problem.
MP_RETURN_IF_ERROR(InternalSolveWeightedOrthogonalProblem(
source_points, target_points, sqrt_weights, transform_mat))
<< "Failed to solve the WEOP problem!";
return mediapipe::OkStatus();
}
private:
static constexpr float kAbsoluteErrorEps = 1e-9f;
static mediapipe::Status ValidateInputPoints(
const Eigen::Matrix3Xf& source_points,
const Eigen::Matrix3Xf& target_points) {
RET_CHECK_GT(source_points.cols(), 0)
<< "The number of source points must be positive!";
RET_CHECK_EQ(source_points.cols(), target_points.cols())
<< "The number of source and target points must be equal!";
return mediapipe::OkStatus();
}
static mediapipe::Status ValidatePointWeights(
int num_points, const Eigen::VectorXf& point_weights) {
RET_CHECK_GT(point_weights.size(), 0)
<< "The number of point weights must be positive!";
RET_CHECK_EQ(point_weights.size(), num_points)
<< "The number of points and point weights must be equal!";
float total_weight = 0.f;
for (int i = 0; i < num_points; ++i) {
RET_CHECK_GE(point_weights(i), 0.f)
<< "Each point weight must be non-negative!";
total_weight += point_weights(i);
}
RET_CHECK_GT(total_weight, kAbsoluteErrorEps)
<< "The total point weight is too small!";
return mediapipe::OkStatus();
}
static Eigen::VectorXf ExtractSquareRoot(
const Eigen::VectorXf& point_weights) {
Eigen::VectorXf sqrt_weights(point_weights);
for (int i = 0; i < sqrt_weights.size(); ++i) {
sqrt_weights(i) = std::sqrt(sqrt_weights(i));
}
return sqrt_weights;
}
// Combines a 3x3 rotation-and-scale matrix and a 3x1 translation vector into
// a single 4x4 transformation matrix.
static Eigen::Matrix4f CombineTransformMatrix(const Eigen::Matrix3f& r_and_s,
const Eigen::Vector3f& t) {
Eigen::Matrix4f result = Eigen::Matrix4f::Identity();
result.leftCols(3).topRows(3) = r_and_s;
result.col(3).topRows(3) = t;
return result;
}
// The weighted problem is thoroughly addressed in Section 2.4 of:
// D. Akca, Generalized Procrustes analysis and its applications
// in photogrammetry, 2003, https://doi.org/10.3929/ethz-a-004656648
//
// Notable differences in the code presented here are:
//
// * In the paper, the weights matrix W_p is Cholesky-decomposed as Q^T Q.
// Our W_p is diagonal (equal to diag(sqrt_weights^2)),
// so we can just set Q = diag(sqrt_weights) instead.
//
// * In the paper, the problem is presented as
// (for W_k = I and W_p = tranposed(Q) Q):
// || Q (c A T + j tranposed(t) - B) || -> min.
//
// We reformulate it as an equivalent minimization of the transpose's
// norm:
// || (c tranposed(T) tranposed(A) - tranposed(B)) tranposed(Q) || -> min,
// where tranposed(A) and tranposed(B) are the source and the target point
// clouds, respectively, c tranposed(T) is the rotation+scaling R sought
// for, and Q is diag(sqrt_weights).
//
// Most of the derivations are therefore transposed.
//
// Note: the output `transform_mat` argument is used instead of `StatusOr<>`
// return type in order to avoid Eigen memory alignment issues. Details:
// https://eigen.tuxfamily.org/dox/group__TopicStructHavingEigenMembers.html
static mediapipe::Status InternalSolveWeightedOrthogonalProblem(
const Eigen::Matrix3Xf& sources, const Eigen::Matrix3Xf& targets,
const Eigen::VectorXf& sqrt_weights, Eigen::Matrix4f& transform_mat) {
// tranposed(A_w).
Eigen::Matrix3Xf weighted_sources =
sources.array().rowwise() * sqrt_weights.array().transpose();
// tranposed(B_w).
Eigen::Matrix3Xf weighted_targets =
targets.array().rowwise() * sqrt_weights.array().transpose();
// w = tranposed(j_w) j_w.
float total_weight = sqrt_weights.cwiseProduct(sqrt_weights).sum();
// Let C = (j_w tranposed(j_w)) / (tranposed(j_w) j_w).
// Note that C = tranposed(C), hence (I - C) = tranposed(I - C).
//
// tranposed(A_w) C = tranposed(A_w) j_w tranposed(j_w) / w =
// (tranposed(A_w) j_w) tranposed(j_w) / w = c_w tranposed(j_w),
//
// where c_w = tranposed(A_w) j_w / w is a k x 1 vector calculated here:
Eigen::Matrix3Xf twice_weighted_sources =
weighted_sources.array().rowwise() * sqrt_weights.array().transpose();
Eigen::Vector3f source_center_of_mass =
twice_weighted_sources.rowwise().sum() / total_weight;
// tranposed((I - C) A_w) = tranposed(A_w) (I - C) =
// tranposed(A_w) - tranposed(A_w) C = tranposed(A_w) - c_w tranposed(j_w).
Eigen::Matrix3Xf centered_weighted_sources =
weighted_sources - source_center_of_mass * sqrt_weights.transpose();
Eigen::Matrix3f rotation;
MP_RETURN_IF_ERROR(ComputeOptimalRotation(
weighted_targets * centered_weighted_sources.transpose(), rotation))
<< "Failed to compute the optimal rotation!";
ASSIGN_OR_RETURN(
float scale,
ComputeOptimalScale(centered_weighted_sources, weighted_sources,
weighted_targets, rotation),
_ << "Failed to compute the optimal scale!");
// R = c tranposed(T).
Eigen::Matrix3f rotation_and_scale = scale * rotation;
// Compute optimal translation for the weighted problem.
// tranposed(B_w - c A_w T) = tranposed(B_w) - R tranposed(A_w) in (54).
const auto pointwise_diffs =
weighted_targets - rotation_and_scale * weighted_sources;
// Multiplication by j_w is a respectively weighted column sum.
// (54) from the paper.
const auto weighted_pointwise_diffs =
pointwise_diffs.array().rowwise() * sqrt_weights.array().transpose();
Eigen::Vector3f translation =
weighted_pointwise_diffs.rowwise().sum() / total_weight;
transform_mat = CombineTransformMatrix(rotation_and_scale, translation);
return mediapipe::OkStatus();
}
// `design_matrix` is a transposed LHS of (51) in the paper.
//
// Note: the output `rotation` argument is used instead of `StatusOr<>`
// return type in order to avoid Eigen memory alignment issues. Details:
// https://eigen.tuxfamily.org/dox/group__TopicStructHavingEigenMembers.html
static mediapipe::Status ComputeOptimalRotation(
const Eigen::Matrix3f& design_matrix, Eigen::Matrix3f& rotation) {
RET_CHECK_GT(design_matrix.norm(), kAbsoluteErrorEps)
<< "Design matrix norm is too small!";
Eigen::JacobiSVD<Eigen::Matrix3f> svd(
design_matrix, Eigen::ComputeFullU | Eigen::ComputeFullV);
Eigen::Matrix3f postrotation = svd.matrixU();
Eigen::Matrix3f prerotation = svd.matrixV().transpose();
// Disallow reflection by ensuring that det(`rotation`) = +1 (and not -1),
// see "4.6 Constrained orthogonal Procrustes problems"
// in the Gower & Dijksterhuis's book "Procrustes Analysis".
// We flip the sign of the least singular value along with a column in W.
//
// Note that now the sum of singular values doesn't work for scale
// estimation due to this sign flip.
if (postrotation.determinant() * prerotation.determinant() <
static_cast<float>(0)) {
postrotation.col(2) *= static_cast<float>(-1);
}
// Transposed (52) from the paper.
rotation = postrotation * prerotation;
return mediapipe::OkStatus();
}
static mediapipe::StatusOr<float> ComputeOptimalScale(
const Eigen::Matrix3Xf& centered_weighted_sources,
const Eigen::Matrix3Xf& weighted_sources,
const Eigen::Matrix3Xf& weighted_targets,
const Eigen::Matrix3f& rotation) {
// tranposed(T) tranposed(A_w) (I - C).
const auto rotated_centered_weighted_sources =
rotation * centered_weighted_sources;
// Use the identity trace(A B) = sum(A * B^T)
// to avoid building large intermediate matrices (* is Hadamard product).
// (53) from the paper.
float numerator =
rotated_centered_weighted_sources.cwiseProduct(weighted_targets).sum();
float denominator =
centered_weighted_sources.cwiseProduct(weighted_sources).sum();
RET_CHECK_GT(denominator, kAbsoluteErrorEps)
<< "Scale expression denominator is too small!";
RET_CHECK_GT(numerator / denominator, kAbsoluteErrorEps)
<< "Scale is too small!";
return numerator / denominator;
}
};
} // namespace
std::unique_ptr<ProcrustesSolver> CreateFloatPrecisionProcrustesSolver() {
return absl::make_unique<FloatPrecisionProcrustesSolver>();
}
} // namespace face_geometry
} // namespace mediapipe
mediapipe/modules/face_geometry/libs/procrustes_solver.h
+70
View File
@@ -0,0 +1,70 @@
// Copyright 2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef MEDIAPIPE_FACE_GEOMETRY_LIBS_PROCRUSTES_SOLVER_H_
#define MEDIAPIPE_FACE_GEOMETRY_LIBS_PROCRUSTES_SOLVER_H_
#include <memory>
#include "Eigen/Dense"
#include "mediapipe/framework/port/status.h"
namespace mediapipe::face_geometry {
// Encapsulates a stateless solver for the Weighted Extended Orthogonal
// Procrustes (WEOP) Problem, as defined in Section 2.4 of
// https://doi.org/10.3929/ethz-a-004656648.
//
// Given the source and the target point clouds, the algorithm estimates
// a 4x4 transformation matrix featuring the following semantic components:
//
// * Uniform scale
// * Rotation
// * Translation
//
// The matrix maps the source point cloud into the target point cloud minimizing
// the Mean Squared Error.
class ProcrustesSolver {
public:
virtual ~ProcrustesSolver() = default;
// Solves the Weighted Extended Orthogonal Procrustes (WEOP) Problem.
//
// All `source_points`, `target_points` and `point_weights` must define the
// same number of points. Elements of `point_weights` must be non-negative.
//
// A too small diameter of either of the point clouds will likely lead to
// numerical instabilities and failure to estimate the transformation.
//
// A too small point cloud total weight will likely lead to numerical
// instabilities and failure to estimate the transformation too.
//
// Small point coordinate deviation for either of the point cloud will likely
// result in a failure as it will make the solution very unstable if possible.
//
// Note: the output `transform_mat` argument is used instead of `StatusOr<>`
// return type in order to avoid Eigen memory alignment issues. Details:
// https://eigen.tuxfamily.org/dox/group__TopicStructHavingEigenMembers.html
virtual mediapipe::Status SolveWeightedOrthogonalProblem(
const Eigen::Matrix3Xf& source_points, //
const Eigen::Matrix3Xf& target_points, //
const Eigen::VectorXf& point_weights, //
Eigen::Matrix4f& transform_mat) const = 0;
};
std::unique_ptr<ProcrustesSolver> CreateFloatPrecisionProcrustesSolver();
} // namespace mediapipe::face_geometry
#endif // MEDIAPIPE_FACE_GEOMETRY_LIBS_PROCRUSTES_SOLVER_H_
mediapipe/modules/face_geometry/libs/validation_utils.cc
+126
View File
@@ -0,0 +1,126 @@
// Copyright 2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/modules/face_geometry/libs/validation_utils.h"
#include <cstdint>
#include <cstdlib>
#include "mediapipe/framework/formats/matrix_data.pb.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/framework/port/status_macros.h"
#include "mediapipe/modules/face_geometry/libs/mesh_3d_utils.h"
#include "mediapipe/modules/face_geometry/protos/environment.pb.h"
#include "mediapipe/modules/face_geometry/protos/geometry_pipeline_metadata.pb.h"
#include "mediapipe/modules/face_geometry/protos/mesh_3d.pb.h"
namespace mediapipe::face_geometry {
mediapipe::Status ValidatePerspectiveCamera(
const PerspectiveCamera& perspective_camera) {
static constexpr float kAbsoluteErrorEps = 1e-9f;
RET_CHECK_GT(perspective_camera.near(), kAbsoluteErrorEps)
<< "Near Z must be greater than 0 with a margin of 10^{-9}!";
RET_CHECK_GT(perspective_camera.far(),
perspective_camera.near() + kAbsoluteErrorEps)
<< "Far Z must be greater than Near Z with a margin of 10^{-9}!";
RET_CHECK_GT(perspective_camera.vertical_fov_degrees(), kAbsoluteErrorEps)
<< "Vertical FOV must be positive with a margin of 10^{-9}!";
RET_CHECK_LT(perspective_camera.vertical_fov_degrees() + kAbsoluteErrorEps,
180.f)
<< "Vertical FOV must be less than 180 degrees with a margin of 10^{-9}";
return mediapipe::OkStatus();
}
mediapipe::Status ValidateEnvironment(const Environment& environment) {
MP_RETURN_IF_ERROR(
ValidatePerspectiveCamera(environment.perspective_camera()))
<< "Invalid perspective camera!";
return mediapipe::OkStatus();
}
mediapipe::Status ValidateMesh3d(const Mesh3d& mesh_3d) {
const std::size_t vertex_size = GetVertexSize(mesh_3d.vertex_type());
const std::size_t primitive_type = GetPrimitiveSize(mesh_3d.primitive_type());
RET_CHECK_EQ(mesh_3d.vertex_buffer_size() % vertex_size, 0)
<< "Vertex buffer size must a multiple of the vertex size!";
RET_CHECK_EQ(mesh_3d.index_buffer_size() % primitive_type, 0)
<< "Index buffer size must a multiple of the primitive size!";
const int num_vertices = mesh_3d.vertex_buffer_size() / vertex_size;
for (uint32_t idx : mesh_3d.index_buffer()) {
RET_CHECK_LT(idx, num_vertices)
<< "All mesh indices must refer to an existing vertex!";
}
return mediapipe::OkStatus();
}
mediapipe::Status ValidateFaceGeometry(const FaceGeometry& face_geometry) {
MP_RETURN_IF_ERROR(ValidateMesh3d(face_geometry.mesh())) << "Invalid mesh!";
static constexpr char kInvalid4x4MatrixMessage[] =
"Pose transformation matrix must be a 4x4 matrix!";
const MatrixData& pose_transform_matrix =
face_geometry.pose_transform_matrix();
RET_CHECK_EQ(pose_transform_matrix.rows(), 4) << kInvalid4x4MatrixMessage;
RET_CHECK_EQ(pose_transform_matrix.rows(), 4) << kInvalid4x4MatrixMessage;
RET_CHECK_EQ(pose_transform_matrix.packed_data_size(), 16)
<< kInvalid4x4MatrixMessage;
return mediapipe::OkStatus();
}
mediapipe::Status ValidateGeometryPipelineMetadata(
const GeometryPipelineMetadata& metadata) {
MP_RETURN_IF_ERROR(ValidateMesh3d(metadata.canonical_mesh()))
<< "Invalid canonical mesh!";
RET_CHECK_GT(metadata.procrustes_landmark_basis_size(), 0)
<< "Procrustes landmark basis must be non-empty!";
const int num_vertices =
metadata.canonical_mesh().vertex_buffer_size() /
GetVertexSize(metadata.canonical_mesh().vertex_type());
for (const WeightedLandmarkRef& wlr : metadata.procrustes_landmark_basis()) {
RET_CHECK_LT(wlr.landmark_id(), num_vertices)
<< "All Procrustes basis indices must refer to an existing canonical "
"mesh vertex!";
RET_CHECK_GE(wlr.weight(), 0.f)
<< "All Procrustes basis landmarks must have a non-negative weight!";
}
return mediapipe::OkStatus();
}
mediapipe::Status ValidateFrameDimensions(int frame_width, int frame_height) {
RET_CHECK_GT(frame_width, 0) << "Frame width must be positive!";
RET_CHECK_GT(frame_height, 0) << "Frame height must be positive!";
return mediapipe::OkStatus();
}
} // namespace mediapipe::face_geometry
mediapipe/modules/face_geometry/libs/validation_utils.h
+70
View File
@@ -0,0 +1,70 @@
// Copyright 2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef MEDIAPIPE_FACE_GEOMETRY_LIBS_VALIDATION_UTILS_H_
#define MEDIAPIPE_FACE_GEOMETRY_LIBS_VALIDATION_UTILS_H_
#include "mediapipe/framework/port/status.h"
#include "mediapipe/modules/face_geometry/protos/environment.pb.h"
#include "mediapipe/modules/face_geometry/protos/face_geometry.pb.h"
#include "mediapipe/modules/face_geometry/protos/geometry_pipeline_metadata.pb.h"
#include "mediapipe/modules/face_geometry/protos/mesh_3d.pb.h"
namespace mediapipe::face_geometry {
// Validates `perspective_camera`.
//
// Near Z must be greater than 0 with a margin of `1e-9`.
// Far Z must be greater than Near Z with a margin of `1e-9`.
// Vertical FOV must be in range (0, 180) with a margin of `1e-9` on the range
// edges.
mediapipe::Status ValidatePerspectiveCamera(
const PerspectiveCamera& perspective_camera);
// Validates `environment`.
//
// Environment's perspective camera must be valid.
mediapipe::Status ValidateEnvironment(const Environment& environment);
// Validates `mesh_3d`.
//
// Mesh vertex buffer size must a multiple of the vertex size.
// Mesh index buffer size must a multiple of the primitive size.
// All mesh indices must reference an existing mesh vertex.
mediapipe::Status ValidateMesh3d(const Mesh3d& mesh_3d);
// Validates `face_geometry`.
//
// Face mesh must be valid.
// Face pose transformation matrix must be a 4x4 matrix.
mediapipe::Status ValidateFaceGeometry(const FaceGeometry& face_geometry);
// Validates `metadata`.
//
// Canonical face mesh must be valid.
// Procrustes landmark basis must be non-empty.
// All Procrustes basis indices must reference an existing canonical mesh
// vertex.
// All Procrustes basis landmarks must have a non-negative weight.
mediapipe::Status ValidateGeometryPipelineMetadata(
const GeometryPipelineMetadata& metadata);
// Validates frame dimensions.
//
// Both frame width and frame height must be positive.
mediapipe::Status ValidateFrameDimensions(int frame_width, int frame_height);
} // namespace mediapipe::face_geometry
#endif // MEDIAPIPE_FACE_GEOMETRY_LIBS_VALIDATION_UTILS_H_
mediapipe/modules/face_geometry/protos/BUILD
+46
View File
@@ -0,0 +1,46 @@
# Copyright 2020 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library")
licenses(["notice"])
package(default_visibility = ["//visibility:public"])
mediapipe_proto_library(
name = "environment_proto",
srcs = ["environment.proto"],
)
mediapipe_proto_library(
name = "face_geometry_proto",
srcs = ["face_geometry.proto"],
deps = [
":mesh_3d_proto",
"//mediapipe/framework/formats:matrix_data_proto",
],
)
mediapipe_proto_library(
name = "geometry_pipeline_metadata_proto",
srcs = ["geometry_pipeline_metadata.proto"],
deps = [
":mesh_3d_proto",
],
)
mediapipe_proto_library(
name = "mesh_3d_proto",
srcs = ["mesh_3d.proto"],
)
mediapipe/modules/face_geometry/protos/environment.proto
+84
View File
@@ -0,0 +1,84 @@
// Copyright 2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto2";
package mediapipe.face_geometry;
option java_package = "com.google.mediapipe.modules.facegeometry";
option java_outer_classname = "EnvironmentProto";
// Defines the (0, 0) origin point location of the environment.
//
// The variation in the origin point location can be traced back to the memory
// layout of the camera video frame buffers.
//
// Usually, the memory layout for most CPU (and also some GPU) camera video
// frame buffers results in having the (0, 0) origin point located in the
// Top Left corner.
//
// On the contrary, the memory layout for most GPU camera video frame buffers
// results in having the (0, 0) origin point located in the Bottom Left corner.
//
// Let's consider the following example:
//
// (A) ---------------+
// ___ |
// | (1) | | |
// | / \ | | |
// | |---|===|-| |
// | |---| | | |
// | / \ | | |
// | | | | | |
// | | (2) |=| | |
// | | | | | |
// | |_______| |_| |
// | |@| |@| | | |
// | ___________|_|_ |
// |
// (B) ---------------+
//
// On this example, (1) and (2) have the same X coordinate regardless of the
// origin point location. However, having the origin point located at (A)
// (Top Left corner) results in (1) having a smaller Y coordinate if compared to
// (2). Similarly, having the origin point located at (B) (Bottom Left corner)
// results in (1) having a greater Y coordinate if compared to (2).
//
// Providing the correct origin point location for your environment and making
// sure all the input landmarks are in-sync with this location is crucial
// for receiving the correct output face geometry and visual renders.
enum OriginPointLocation {
BOTTOM_LEFT_CORNER = 1;
TOP_LEFT_CORNER = 2;
}
// The perspective camera is defined through its vertical FOV angle and the
// Z-clipping planes. The aspect ratio is a runtime variable for the face
// geometry module and should be provided alongside the face landmarks in order
// to estimate the face geometry on a given frame.
//
// More info on Perspective Cameras:
// http://www.songho.ca/opengl/gl_projectionmatrix.html#perspective
message PerspectiveCamera {
// `0 < vertical_fov_degrees < 180`.
optional float vertical_fov_degrees = 1;
// `0 < near < far`.
optional float near = 2;
optional float far = 3;
}
message Environment {
optional OriginPointLocation origin_point_location = 1;
optional PerspectiveCamera perspective_camera = 2;
}
mediapipe/modules/face_geometry/protos/face_geometry.proto
+51
View File
@@ -0,0 +1,51 @@
// Copyright 2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto2";
package mediapipe.face_geometry;
import "mediapipe/framework/formats/matrix_data.proto";
import "mediapipe/modules/face_geometry/protos/mesh_3d.proto";
option java_package = "com.google.mediapipe.modules.facegeometry";
option java_outer_classname = "FaceGeometryProto";
// Defines the face geometry pipeline estimation result format.
message FaceGeometry {
// Defines a mesh surface for a face. The face mesh vertex IDs are the same as
// the face landmark IDs.
//
// XYZ coordinates exist in the right-handed Metric 3D space configured by an
// environment. UV coodinates are taken from the canonical face mesh model.
//
// XY coordinates are guaranteed to match the screen positions of
// the input face landmarks after (1) being multiplied by the face pose
// transformation matrix and then (2) being projected with a perspective
// camera matrix of the same environment.
optional Mesh3d mesh = 1;
// Defines a face pose transformation matrix, which provides mapping from
// the static canonical face model to the runtime face. Tries to distinguish
// a head pose change from a facial expression change and to only reflect the
// former.
//
// Is a 4x4 matrix and contains only the following components:
// * Uniform scale
// * Rotation
// * Translation
//
// The last row is guaranteed to be `[0 0 0 1]`.
optional MatrixData pose_transform_matrix = 2;
}
mediapipe/modules/face_geometry/protos/geometry_pipeline_metadata.proto
+46
View File
@@ -0,0 +1,46 @@
// Copyright 2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto2";
package mediapipe.face_geometry;
import "mediapipe/modules/face_geometry/protos/mesh_3d.proto";
option java_package = "com.google.mediapipe.modules.facegeometry";
option java_outer_classname = "GeometryPipelineMetadataProto";
message WeightedLandmarkRef {
// Defines the landmark ID. References an existing face landmark ID.
optional uint32 landmark_id = 1;
// Defines the landmark weight. The larger the weight the more influence this
// landmark has in the basis.
//
// Is positive.
optional float weight = 2;
}
message GeometryPipelineMetadata {
// Defines a mesh surface for a canonical face. The canonical face mesh vertex
// IDs are the same as the face landmark IDs.
//
// XYZ coordinates are defined in centimeter units.
optional Mesh3d canonical_mesh = 1;
// Defines a weighted landmark basis for running the Procrustes solver
// algorithm inside the geometry pipeline.
//
// A good basis sets face landmark weights in way to distinguish a head pose
// change from a facial expression change and to only respond to the former.
repeated WeightedLandmarkRef procrustes_landmark_basis = 2;
}
mediapipe/modules/face_geometry/protos/mesh_3d.proto
+41
View File
@@ -0,0 +1,41 @@
// Copyright 2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto2";
package mediapipe.face_geometry;
option java_package = "com.google.mediapipe.modules.facegeometry";
option java_outer_classname = "Mesh3dProto";
message Mesh3d {
enum VertexType {
// Is defined by 5 coordinates: Position (XYZ) + Texture coordinate (UV).
VERTEX_PT = 0;
}
enum PrimitiveType {
// Is defined by 3 indices: triangle vertex IDs.
TRIANGLE = 0;
}
optional VertexType vertex_type = 1;
optional PrimitiveType primitive_type = 2;
// Vertex buffer size is a multiple of the vertex size (e.g., 5 for
// VERTEX_PT).
repeated float vertex_buffer = 3;
// Index buffer size is a multiple of the primitive size (e.g., 3 for
// TRIANGLE).
repeated uint32 index_buffer = 4;
}
mediapipe/python/BUILD
+1
View File
@@ -49,5 +49,6 @@ pybind_extension(
"//mediapipe/python/pybind:packet_getter",
"//mediapipe/python/pybind:resource_util",
"//mediapipe/python/pybind:timestamp",
"//mediapipe/python/pybind:validated_graph_config",
],
)

Some files were not shown because too many files have changed in this diff Show More