talksik/mediapipe
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Project import generated by Copybara.

Browse Source 
GitOrigin-RevId: e3566e5029af25b0fc4b1071a49e49ae20aa5df6
This commit is contained in:
MediaPipe Team
2019-12-02 17:54:10 -08:00
committed by jqtang
parent 446d7cf6b6
commit 137867d088
74 changed files with 909 additions and 504 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mediapipe/calculators/core/BUILD
+1
View File
@@ -691,6 +691,7 @@ cc_library(
":split_vector_calculator_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/formats:landmark_cc_proto",
"//mediapipe/framework/formats:rect_cc_proto",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"//mediapipe/util:resource_util",
mediapipe/calculators/core/begin_loop_calculator.cc
+4 -10
View File
@@ -21,16 +21,10 @@
namespace mediapipe {
// A calculator to process std::vector<NormalizedLandmark>.
typedef BeginLoopCalculator<std::vector<::mediapipe::NormalizedLandmark>>
BeginLoopNormalizedLandmarkCalculator;
REGISTER_CALCULATOR(BeginLoopNormalizedLandmarkCalculator);
// A calculator to process std::vector<std::vector<NormalizedLandmark>>.
typedef BeginLoopCalculator<
std::vector<std::vector<::mediapipe::NormalizedLandmark>>>
BeginLoopNormalizedLandmarksVectorCalculator;
REGISTER_CALCULATOR(BeginLoopNormalizedLandmarksVectorCalculator);
// A calculator to process std::vector<NormalizedLandmarkList>.
typedef BeginLoopCalculator<std::vector<::mediapipe::NormalizedLandmarkList>>
BeginLoopNormalizedLandmarkListVectorCalculator;
REGISTER_CALCULATOR(BeginLoopNormalizedLandmarkListVectorCalculator);
// A calculator to process std::vector<NormalizedRect>.
typedef BeginLoopCalculator<std::vector<::mediapipe::NormalizedRect>>
mediapipe/calculators/core/concatenate_vector_calculator.cc
+2 -2
View File
@@ -19,7 +19,7 @@
#include "mediapipe/framework/formats/landmark.pb.h"
#include "tensorflow/lite/interpreter.h"
#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__APPLE__)
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
#include "tensorflow/lite/delegates/gpu/gl/gl_buffer.h"
#endif // !MEDIAPIPE_DISABLE_GPU
@@ -50,7 +50,7 @@ typedef ConcatenateVectorCalculator<::mediapipe::NormalizedLandmark>
ConcatenateLandmarkVectorCalculator;
REGISTER_CALCULATOR(ConcatenateLandmarkVectorCalculator);
#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__APPLE__)
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
typedef ConcatenateVectorCalculator<::tflite::gpu::gl::GlBuffer>
ConcatenateGlBufferVectorCalculator;
REGISTER_CALCULATOR(ConcatenateGlBufferVectorCalculator);
mediapipe/calculators/core/end_loop_calculator.cc
+3 -8
View File
@@ -26,14 +26,9 @@ typedef EndLoopCalculator<std::vector<::mediapipe::NormalizedRect>>
EndLoopNormalizedRectCalculator;
REGISTER_CALCULATOR(EndLoopNormalizedRectCalculator);
typedef EndLoopCalculator<std::vector<::mediapipe::NormalizedLandmark>>
EndLoopNormalizedLandmarkCalculator;
REGISTER_CALCULATOR(EndLoopNormalizedLandmarkCalculator);
typedef EndLoopCalculator<
std::vector<std::vector<::mediapipe::NormalizedLandmark>>>
EndLoopNormalizedLandmarksVectorCalculator;
REGISTER_CALCULATOR(EndLoopNormalizedLandmarksVectorCalculator);
typedef EndLoopCalculator<std::vector<::mediapipe::NormalizedLandmarkList>>
EndLoopNormalizedLandmarkListVectorCalculator;
REGISTER_CALCULATOR(EndLoopNormalizedLandmarkListVectorCalculator);
typedef EndLoopCalculator<std::vector<bool>> EndLoopBooleanCalculator;
REGISTER_CALCULATOR(EndLoopBooleanCalculator);
mediapipe/calculators/core/previous_loopback_calculator.cc
+4 -17
View File
@@ -74,6 +74,10 @@ class PreviousLoopbackCalculator : public CalculatorBase {
}
::mediapipe::Status Process(CalculatorContext* cc) final {
Packet& main_packet = cc->Inputs().Get(main_id_).Value();
if (!main_packet.IsEmpty()) {
main_ts_.push_back(main_packet.Timestamp());
}
Packet& loopback_packet = cc->Inputs().Get(loop_id_).Value();
if (!loopback_packet.IsEmpty()) {
loopback_packets_.push_back(loopback_packet);
@@ -83,23 +87,6 @@ class PreviousLoopbackCalculator : public CalculatorBase {
}
}
Packet& main_packet = cc->Inputs().Get(main_id_).Value();
if (!main_packet.IsEmpty()) {
main_ts_.push_back(main_packet.Timestamp());
// In case of an empty "LOOP" input, truncate timestamp is set to the
// lowest possible timestamp for a successive non-empty "LOOP" input. This
// truncates main_ts_ as soon as possible, and produces the highest legal
// output timestamp bound.
if (loopback_packet.IsEmpty() &&
loopback_packet.Timestamp() != Timestamp::Unstarted()) {
while (!main_ts_.empty() &&
main_ts_.front() <= loopback_packet.Timestamp() + 1) {
main_ts_.pop_front();
}
}
}
while (!main_ts_.empty() && !loopback_packets_.empty()) {
Timestamp main_timestamp = main_ts_.front();
main_ts_.pop_front();
mediapipe/calculators/core/previous_loopback_calculator_test.cc
+15 -65
View File
@@ -93,14 +93,19 @@ TEST(PreviousLoopbackCalculator, CorrectTimestamps) {
EXPECT_EQ(TimestampValues(in_prev), (std::vector<int64>{1}));
EXPECT_EQ(pair_values(in_prev.back()), std::make_pair(1, -1));
send_packet("in", 2);
MP_EXPECT_OK(graph_.WaitUntilIdle());
EXPECT_EQ(TimestampValues(in_prev), (std::vector<int64>{1, 2}));
EXPECT_EQ(pair_values(in_prev.back()), std::make_pair(2, 1));
send_packet("in", 5);
MP_EXPECT_OK(graph_.WaitUntilIdle());
EXPECT_EQ(TimestampValues(in_prev), (std::vector<int64>{1, 5}));
EXPECT_EQ(pair_values(in_prev.back()), std::make_pair(5, 1));
EXPECT_EQ(TimestampValues(in_prev), (std::vector<int64>{1, 2, 5}));
EXPECT_EQ(pair_values(in_prev.back()), std::make_pair(5, 2));
send_packet("in", 15);
MP_EXPECT_OK(graph_.WaitUntilIdle());
EXPECT_EQ(TimestampValues(in_prev), (std::vector<int64>{1, 5, 15}));
EXPECT_EQ(TimestampValues(in_prev), (std::vector<int64>{1, 2, 5, 15}));
EXPECT_EQ(pair_values(in_prev.back()), std::make_pair(15, 5));
MP_EXPECT_OK(graph_.CloseAllInputStreams());
@@ -182,80 +187,25 @@ TEST(PreviousLoopbackCalculator, ClosesCorrectly) {
MP_EXPECT_OK(graph_.WaitUntilIdle());
EXPECT_EQ(TimestampValues(outputs), (std::vector<int64>{1}));
send_packet("in", 2);
MP_EXPECT_OK(graph_.WaitUntilIdle());
EXPECT_EQ(TimestampValues(outputs), (std::vector<int64>{1, 2}));
send_packet("in", 5);
MP_EXPECT_OK(graph_.WaitUntilIdle());
EXPECT_EQ(TimestampValues(outputs), (std::vector<int64>{1, 5}));
EXPECT_EQ(TimestampValues(outputs), (std::vector<int64>{1, 2, 5}));
send_packet("in", 15);
MP_EXPECT_OK(graph_.WaitUntilIdle());
EXPECT_EQ(TimestampValues(outputs), (std::vector<int64>{1, 5, 15}));
EXPECT_EQ(TimestampValues(outputs), (std::vector<int64>{1, 2, 5, 15}));
MP_EXPECT_OK(graph_.CloseAllInputStreams());
MP_EXPECT_OK(graph_.WaitUntilIdle());
EXPECT_EQ(TimestampValues(outputs),
(std::vector<int64>{1, 5, 15, Timestamp::Max().Value()}));
(std::vector<int64>{1, 2, 5, 15, Timestamp::Max().Value()}));
MP_EXPECT_OK(graph_.WaitUntilDone());
}
// Demonstrates that downstream calculators won't be blocked by
// always-empty-LOOP-stream.
TEST(PreviousLoopbackCalculator, EmptyLoopForever) {
std::vector<Packet> outputs;
CalculatorGraphConfig graph_config_ =
ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
input_stream: 'in'
node {
calculator: 'PreviousLoopbackCalculator'
input_stream: 'MAIN:in'
input_stream: 'LOOP:previous'
input_stream_info: { tag_index: 'LOOP' back_edge: true }
output_stream: 'PREV_LOOP:previous'
}
# This calculator synchronizes its inputs as normal, so it is used
# to check that both "in" and "previous" are ready.
node {
calculator: 'PassThroughCalculator'
input_stream: 'in'
input_stream: 'previous'
output_stream: 'out'
output_stream: 'previous2'
}
node {
calculator: 'PacketOnCloseCalculator'
input_stream: 'out'
output_stream: 'close_out'
}
)");
tool::AddVectorSink("close_out", &graph_config_, &outputs);
CalculatorGraph graph_;
MP_ASSERT_OK(graph_.Initialize(graph_config_, {}));
MP_ASSERT_OK(graph_.StartRun({}));
auto send_packet = [&graph_](const std::string& input_name, int n) {
MP_EXPECT_OK(graph_.AddPacketToInputStream(
input_name, MakePacket<int>(n).At(Timestamp(n))));
};
send_packet("in", 0);
MP_EXPECT_OK(graph_.WaitUntilIdle());
EXPECT_EQ(TimestampValues(outputs), (std::vector<int64>{0}));
for (int main_ts = 1; main_ts < 50; ++main_ts) {
send_packet("in", main_ts);
MP_EXPECT_OK(graph_.WaitUntilIdle());
std::vector<int64> ts_values = TimestampValues(outputs);
EXPECT_EQ(ts_values.size(), main_ts);
for (int j = 0; j < main_ts; ++j) {
CHECK_EQ(ts_values[j], j);
}
}
MP_EXPECT_OK(graph_.CloseAllInputStreams());
MP_EXPECT_OK(graph_.WaitUntilIdle());
MP_EXPECT_OK(graph_.WaitUntilDone());
}
} // anonymous namespace
} // namespace mediapipe
mediapipe/calculators/core/split_vector_calculator.cc
+5
View File
@@ -17,6 +17,7 @@
#include <vector>
#include "mediapipe/framework/formats/landmark.pb.h"
#include "mediapipe/framework/formats/rect.pb.h"
#include "tensorflow/lite/interpreter.h"
namespace mediapipe {
@@ -41,4 +42,8 @@ REGISTER_CALCULATOR(SplitTfLiteTensorVectorCalculator);
typedef SplitVectorCalculator<::mediapipe::NormalizedLandmark>
SplitLandmarkVectorCalculator;
REGISTER_CALCULATOR(SplitLandmarkVectorCalculator);
typedef SplitVectorCalculator<::mediapipe::NormalizedRect>
SplitNormalizedRectVectorCalculator;
REGISTER_CALCULATOR(SplitNormalizedRectVectorCalculator);
} // namespace mediapipe
mediapipe/calculators/image/BUILD
+1 -1
View File
@@ -356,13 +356,13 @@ cc_library(
"//mediapipe/framework/port:opencv_imgproc",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"//mediapipe/gpu:gpu_buffer",
] + select({
"//mediapipe/gpu:disable_gpu": [],
"//conditions:default": [
"//mediapipe/gpu:gl_calculator_helper",
"//mediapipe/gpu:gl_simple_shaders",
"//mediapipe/gpu:gl_quad_renderer",
"//mediapipe/gpu:gpu_buffer",
"//mediapipe/gpu:shader_util",
],
}),
mediapipe/calculators/image/image_transformation_calculator.cc
+1 -1
View File
@@ -400,7 +400,7 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
QuadRenderer* renderer = nullptr;
GlTexture src1;
#if defined(__APPLE__) && !TARGET_OS_OSX
#if defined(MEDIAPIPE_IOS)
if (input.format() == GpuBufferFormat::kBiPlanar420YpCbCr8VideoRange ||
input.format() == GpuBufferFormat::kBiPlanar420YpCbCr8FullRange) {
if (!yuv_renderer_) {
mediapipe/calculators/tensorflow/tensorflow_inference_calculator.cc
+2 -2
View File
@@ -34,7 +34,7 @@
#include "tensorflow/core/framework/tensor_shape.h"
#include "tensorflow/core/framework/tensor_util.h"
#if !defined(__ANDROID__) && !defined(__APPLE__)
#if !defined(MEDIAPIPE_MOBILE) && !defined(__APPLE__)
#include "tensorflow/core/profiler/lib/traceme.h"
#endif
@@ -441,7 +441,7 @@ class TensorFlowInferenceCalculator : public CalculatorBase {
const int64 run_start_time = absl::ToUnixMicros(clock_->TimeNow());
tf::Status tf_status;
{
#if !defined(__ANDROID__) && !defined(__APPLE__)
#if !defined(MEDIAPIPE_MOBILE) && !defined(__APPLE__)
tensorflow::profiler::TraceMe trace(absl::string_view(cc->NodeName()));
#endif
tf_status = session_->Run(input_tensors, output_tensor_names,
mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_calculator.cc
+1 -2
View File
@@ -31,8 +31,7 @@
#include "mediapipe/framework/tool/status_util.h"
#include "tensorflow/core/public/session_options.h"
#if defined(MEDIAPIPE_LITE) || defined(__ANDROID__) || \
defined(__APPLE__) && !TARGET_OS_OSX
#if defined(MEDIAPIPE_MOBILE)
#include "mediapipe/util/android/file/base/helpers.h"
#else
#include "mediapipe/framework/port/file_helpers.h"
mediapipe/calculators/tflite/BUILD
+7 -2
View File
@@ -13,12 +13,12 @@
# limitations under the License.
#
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
licenses(["notice"]) # Apache 2.0
package(default_visibility = ["//visibility:private"])
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
proto_library(
name = "ssd_anchors_calculator_proto",
srcs = ["ssd_anchors_calculator.proto"],
@@ -249,6 +249,11 @@ cc_library(
"@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_program",
"@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_shader",
],
}) + select({
"//conditions:default": [],
"//mediapipe:android": [
"@org_tensorflow//tensorflow/lite/delegates/nnapi:nnapi_delegate",
],
}),
alwayslink = 1,
)
mediapipe/calculators/tflite/tflite_converter_calculator.cc
+21 -28
View File
@@ -25,8 +25,7 @@
#include "tensorflow/lite/error_reporter.h"
#include "tensorflow/lite/interpreter.h"
#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
!defined(__APPLE__)
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
#include "mediapipe/gpu/gl_calculator_helper.h"
#include "mediapipe/gpu/gpu_buffer.h"
#include "tensorflow/lite/delegates/gpu/gl/gl_buffer.h"
@@ -35,7 +34,7 @@
#include "tensorflow/lite/delegates/gpu/gl_delegate.h"
#endif // !MEDIAPIPE_DISABLE_GPU
#if defined(__APPLE__) && !TARGET_OS_OSX // iOS
#if defined(MEDIAPIPE_IOS)
#import <CoreVideo/CoreVideo.h>
#import <Metal/Metal.h>
#import <MetalKit/MetalKit.h>
@@ -46,10 +45,9 @@
#include "tensorflow/lite/delegates/gpu/metal_delegate.h"
#endif // iOS
#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
!defined(__APPLE__)
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
typedef ::tflite::gpu::gl::GlBuffer GpuTensor;
#elif defined(__APPLE__) && !TARGET_OS_OSX // iOS
#elif defined(MEDIAPIPE_IOS)
typedef id<MTLBuffer> GpuTensor;
#endif
@@ -69,8 +67,7 @@ typedef Eigen::Matrix<float, Eigen::Dynamic, Eigen::Dynamic, Eigen::ColMajor>
namespace mediapipe {
#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
!defined(__APPLE__)
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
using ::tflite::gpu::gl::CreateReadWriteShaderStorageBuffer;
using ::tflite::gpu::gl::GlProgram;
using ::tflite::gpu::gl::GlShader;
@@ -80,7 +77,7 @@ struct GPUData {
GlShader shader;
GlProgram program;
};
#elif defined(__APPLE__) && !TARGET_OS_OSX // iOS
#elif defined(MEDIAPIPE_IOS)
struct GPUData {
int elements = 1;
GpuTensor buffer;
@@ -149,11 +146,10 @@ class TfLiteConverterCalculator : public CalculatorBase {
std::unique_ptr<tflite::Interpreter> interpreter_ = nullptr;
#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
!defined(__APPLE__)
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
mediapipe::GlCalculatorHelper gpu_helper_;
std::unique_ptr<GPUData> gpu_data_out_;
#elif defined(__APPLE__) && !TARGET_OS_OSX // iOS
#elif defined(MEDIAPIPE_IOS)
MPPMetalHelper* gpu_helper_ = nullptr;
std::unique_ptr<GPUData> gpu_data_out_;
#endif
@@ -202,10 +198,9 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
#endif // !MEDIAPIPE_DISABLE_GPU
if (use_gpu) {
#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
!defined(__APPLE__)
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc));
#elif defined(__APPLE__) && !TARGET_OS_OSX // iOS
#elif defined(MEDIAPIPE_IOS)
MP_RETURN_IF_ERROR([MPPMetalHelper updateContract:cc]);
#endif
}
@@ -236,10 +231,9 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
cc->Outputs().HasTag("TENSORS_GPU"));
// Cannot use quantization.
use_quantized_tensors_ = false;
#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
!defined(__APPLE__)
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
MP_RETURN_IF_ERROR(gpu_helper_.Open(cc));
#elif defined(__APPLE__) && !TARGET_OS_OSX // iOS
#elif defined(MEDIAPIPE_IOS)
gpu_helper_ = [[MPPMetalHelper alloc] initWithCalculatorContext:cc];
RET_CHECK(gpu_helper_);
#endif
@@ -270,11 +264,10 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
}
::mediapipe::Status TfLiteConverterCalculator::Close(CalculatorContext* cc) {
#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
!defined(__APPLE__)
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
gpu_helper_.RunInGlContext([this] { gpu_data_out_.reset(); });
#endif
#if defined(__APPLE__) && !TARGET_OS_OSX // iOS
#if defined(MEDIAPIPE_IOS)
gpu_data_out_.reset();
#endif
return ::mediapipe::OkStatus();
@@ -390,8 +383,7 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
::mediapipe::Status TfLiteConverterCalculator::ProcessGPU(
CalculatorContext* cc) {
#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
!defined(__APPLE__)
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
// GpuBuffer to tflite::gpu::GlBuffer conversion.
const auto& input = cc->Inputs().Tag("IMAGE_GPU").Get<mediapipe::GpuBuffer>();
MP_RETURN_IF_ERROR(
@@ -427,7 +419,7 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
cc->Outputs()
.Tag("TENSORS_GPU")
.Add(output_tensors.release(), cc->InputTimestamp());
#elif defined(__APPLE__) && !TARGET_OS_OSX // iOS
#elif defined(MEDIAPIPE_IOS)
// GpuBuffer to id<MTLBuffer> conversion.
const auto& input = cc->Inputs().Tag("IMAGE_GPU").Get<mediapipe::GpuBuffer>();
{
@@ -493,8 +485,7 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
RET_CHECK_FAIL() << "Num input channels is less than desired output.";
#endif // !MEDIAPIPE_DISABLE_GPU
#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
!defined(__APPLE__)
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext(
[this, &include_alpha, &input, &single_channel]() -> ::mediapipe::Status {
// Device memory.
@@ -538,7 +529,9 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
&gpu_data_out_->program));
return ::mediapipe::OkStatus();
}));
#elif defined(__APPLE__) && !TARGET_OS_OSX // iOS
#elif defined(MEDIAPIPE_IOS)
RET_CHECK(include_alpha)
<< "iOS GPU inference currently accepts only RGBA input.";
@@ -619,7 +612,7 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
CHECK_GE(max_num_channels_, 1);
CHECK_LE(max_num_channels_, 4);
CHECK_NE(max_num_channels_, 2);
#if defined(__APPLE__) && !TARGET_OS_OSX // iOS
#if defined(MEDIAPIPE_IOS)
if (cc->Inputs().HasTag("IMAGE_GPU"))
// Currently on iOS, tflite gpu input tensor must be 4 channels,
// so input image must be 4 channels also (checked in InitGpu).
mediapipe/calculators/tflite/tflite_inference_calculator.cc
+58 -48
View File
@@ -27,7 +27,7 @@
#include "tensorflow/lite/kernels/register.h"
#include "tensorflow/lite/model.h"
#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
#include "mediapipe/gpu/gl_calculator_helper.h"
#include "mediapipe/gpu/gpu_buffer.h"
#include "tensorflow/lite/delegates/gpu/common/shape.h"
@@ -35,9 +35,9 @@
#include "tensorflow/lite/delegates/gpu/gl/gl_program.h"
#include "tensorflow/lite/delegates/gpu/gl/gl_shader.h"
#include "tensorflow/lite/delegates/gpu/gl_delegate.h"
#endif // !MEDIAPIPE_DISABLE_GPU
#endif // !MEDIAPIPE_DISABLE_GL_COMPUTE
#if defined(__APPLE__) && !TARGET_OS_OSX // iOS
#if defined(MEDIAPIPE_IOS)
#import <CoreVideo/CoreVideo.h>
#import <Metal/Metal.h>
#import <MetalKit/MetalKit.h>
@@ -51,12 +51,15 @@
#include "tensorflow/lite/delegates/gpu/metal_delegate_internal.h"
#endif // iOS
#if defined(MEDIAPIPE_ANDROID)
#include "tensorflow/lite/delegates/nnapi/nnapi_delegate.h"
#endif // ANDROID
namespace {
#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
!defined(__APPLE__)
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
typedef ::tflite::gpu::gl::GlBuffer GpuTensor;
#elif defined(__APPLE__) && !TARGET_OS_OSX // iOS
#elif defined(MEDIAPIPE_IOS)
typedef id<MTLBuffer> GpuTensor;
#endif
@@ -92,8 +95,7 @@ std::unique_ptr<tflite::Interpreter> BuildEdgeTpuInterpreter(
// * Aux
namespace mediapipe {
#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
!defined(__APPLE__)
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
using ::tflite::gpu::gl::CopyBuffer;
using ::tflite::gpu::gl::CreateReadWriteShaderStorageBuffer;
using ::tflite::gpu::gl::GlBuffer;
@@ -172,12 +174,11 @@ class TfLiteInferenceCalculator : public CalculatorBase {
std::unique_ptr<tflite::FlatBufferModel> model_;
TfLiteDelegate* delegate_ = nullptr;
#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
!defined(__APPLE__)
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
mediapipe::GlCalculatorHelper gpu_helper_;
std::unique_ptr<GPUData> gpu_data_in_;
std::vector<std::unique_ptr<GPUData>> gpu_data_out_;
#elif defined(__APPLE__) && !TARGET_OS_OSX // iOS
#elif defined(MEDIAPIPE_IOS)
MPPMetalHelper* gpu_helper_ = nullptr;
std::unique_ptr<GPUData> gpu_data_in_;
std::vector<std::unique_ptr<GPUData>> gpu_data_out_;
@@ -237,10 +238,9 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
use_gpu |= options.use_gpu();
if (use_gpu) {
#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
!defined(__APPLE__)
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc));
#elif defined(__APPLE__) && !TARGET_OS_OSX // iOS
#elif defined(MEDIAPIPE_IOS)
MP_RETURN_IF_ERROR([MPPMetalHelper updateContract:cc]);
#endif
}
@@ -280,26 +280,24 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
MP_RETURN_IF_ERROR(LoadModel(cc));
if (gpu_inference_) {
#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
!defined(__APPLE__)
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
MP_RETURN_IF_ERROR(gpu_helper_.Open(cc));
#elif defined(__APPLE__) && !TARGET_OS_OSX // iOS
#elif defined(MEDIAPIPE_IOS)
gpu_helper_ = [[MPPMetalHelper alloc] initWithCalculatorContext:cc];
RET_CHECK(gpu_helper_);
#endif
#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
!defined(__APPLE__)
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext(
[this, &cc]() -> ::mediapipe::Status { return LoadDelegate(cc); }));
#else
MP_RETURN_IF_ERROR(LoadDelegate(cc));
#endif
} else {
#if defined(__EMSCRIPTEN__) || defined(MEDIAPIPE_ANDROID)
MP_RETURN_IF_ERROR(LoadDelegate(cc));
#endif // __EMSCRIPTEN__ || ANDROID
}
#if defined(__EMSCRIPTEN__)
MP_RETURN_IF_ERROR(LoadDelegate(cc));
#endif // __EMSCRIPTEN__
return ::mediapipe::OkStatus();
}
@@ -307,8 +305,7 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
// 1. Receive pre-processed tensor inputs.
if (gpu_input_) {
// Read GPU input into SSBO.
#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
!defined(__APPLE__)
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
const auto& input_tensors =
cc->Inputs().Tag("TENSORS_GPU").Get<std::vector<GpuTensor>>();
RET_CHECK_EQ(input_tensors.size(), 1);
@@ -318,7 +315,7 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
RET_CHECK_CALL(CopyBuffer(input_tensors[0], gpu_data_in_->buffer));
return ::mediapipe::OkStatus();
}));
#elif defined(__APPLE__) && !TARGET_OS_OSX // iOS
#elif defined(MEDIAPIPE_IOS)
const auto& input_tensors =
cc->Inputs().Tag("TENSORS_GPU").Get<std::vector<GpuTensor>>();
RET_CHECK_EQ(input_tensors.size(), 1);
@@ -354,14 +351,13 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
// 2. Run inference.
if (gpu_inference_) {
#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
!defined(__APPLE__)
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
MP_RETURN_IF_ERROR(
gpu_helper_.RunInGlContext([this]() -> ::mediapipe::Status {
RET_CHECK_EQ(interpreter_->Invoke(), kTfLiteOk);
return ::mediapipe::OkStatus();
}));
#elif defined(__APPLE__) && !TARGET_OS_OSX // iOS
#elif defined(MEDIAPIPE_IOS)
RET_CHECK_EQ(interpreter_->Invoke(), kTfLiteOk);
#endif
} else {
@@ -370,8 +366,7 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
// 3. Output processed tensors.
if (gpu_output_) {
#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
!defined(__APPLE__)
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
// Output result tensors (GPU).
auto output_tensors = absl::make_unique<std::vector<GpuTensor>>();
MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext(
@@ -388,7 +383,7 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
cc->Outputs()
.Tag("TENSORS_GPU")
.Add(output_tensors.release(), cc->InputTimestamp());
#elif defined(__APPLE__) && !TARGET_OS_OSX // iOS
#elif defined(MEDIAPIPE_IOS)
// Output result tensors (GPU).
auto output_tensors = absl::make_unique<std::vector<GpuTensor>>();
output_tensors->resize(gpu_data_out_.size());
@@ -433,23 +428,24 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
::mediapipe::Status TfLiteInferenceCalculator::Close(CalculatorContext* cc) {
if (delegate_) {
#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
!defined(__APPLE__)
MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext([this]() -> Status {
TfLiteGpuDelegateDelete(delegate_);
if (gpu_inference_) {
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext([this]() -> Status {
TfLiteGpuDelegateDelete(delegate_);
gpu_data_in_.reset();
for (int i = 0; i < gpu_data_out_.size(); ++i) {
gpu_data_out_[i].reset();
}
return ::mediapipe::OkStatus();
}));
#elif defined(MEDIAPIPE_IOS)
TFLGpuDelegateDelete(delegate_);
gpu_data_in_.reset();
for (int i = 0; i < gpu_data_out_.size(); ++i) {
gpu_data_out_[i].reset();
}
return ::mediapipe::OkStatus();
}));
#elif defined(__APPLE__) && !TARGET_OS_OSX // iOS
TFLGpuDelegateDelete(delegate_);
gpu_data_in_.reset();
for (int i = 0; i < gpu_data_out_.size(); ++i) {
gpu_data_out_[i].reset();
}
#endif
}
delegate_ = nullptr;
}
#if defined(MEDIAPIPE_EDGE_TPU)
@@ -522,8 +518,22 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
::mediapipe::Status TfLiteInferenceCalculator::LoadDelegate(
CalculatorContext* cc) {
#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
!defined(__APPLE__)
#if defined(MEDIAPIPE_ANDROID)
if (!gpu_inference_) {
if (cc->Options<mediapipe::TfLiteInferenceCalculatorOptions>()
.use_nnapi()) {
// Attempt to use NNAPI.
// If not supported, the default CPU delegate will be created and used.
interpreter_->SetAllowFp16PrecisionForFp32(1);
delegate_ = tflite::NnApiDelegate();
RET_CHECK_EQ(interpreter_->ModifyGraphWithDelegate(delegate_), kTfLiteOk);
}
// Return, no need for GPU delegate below.
return ::mediapipe::OkStatus();
}
#endif // ANDROID
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
// Configure and create the delegate.
TfLiteGpuDelegateOptions options = TfLiteGpuDelegateOptionsDefault();
options.compile_options.precision_loss_allowed = 1;
@@ -583,7 +593,7 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
RET_CHECK_EQ(interpreter_->ModifyGraphWithDelegate(delegate_), kTfLiteOk);
#endif // OpenGL
#if defined(__APPLE__) && !TARGET_OS_OSX // iOS
#if defined(MEDIAPIPE_IOS)
// Configure and create the delegate.
TFLGpuDelegateOptions options;
options.allow_precision_loss = false; // Must match converter, F=float/T=half
mediapipe/calculators/tflite/tflite_inference_calculator.proto
+5
View File
@@ -45,4 +45,9 @@ message TfLiteInferenceCalculatorOptions {
// input tensors are on CPU. For input tensors on GPU, GPU backend is always
// used.
optional bool use_gpu = 2 [default = false];
// Android only. When true, an NNAPI delegate will be used for inference.
// If NNAPI is not available, then the default CPU delegate will be used
// automatically.
optional bool use_nnapi = 3 [default = false];
}
mediapipe/calculators/tflite/tflite_tensors_to_classification_calculator.cc
+1 -2
View File
@@ -24,8 +24,7 @@
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/util/resource_util.h"
#include "tensorflow/lite/interpreter.h"
#if defined(__EMSCRIPTEN__) || defined(__ANDROID__) || \
(defined(__APPLE__) && !TARGET_OS_OSX)
#if defined(MEDIAPIPE_MOBILE)
#include "mediapipe/util/android/file/base/file.h"
#include "mediapipe/util/android/file/base/helpers.h"
#else
mediapipe/calculators/tflite/tflite_tensors_to_detections_calculator.cc
+19 -29
View File
@@ -27,8 +27,7 @@
#include "mediapipe/framework/port/ret_check.h"
#include "tensorflow/lite/interpreter.h"
#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
!defined(__APPLE__)
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
#include "mediapipe/gpu/gl_calculator_helper.h"
#include "tensorflow/lite/delegates/gpu/gl/gl_buffer.h"
#include "tensorflow/lite/delegates/gpu/gl/gl_program.h"
@@ -36,7 +35,7 @@
#include "tensorflow/lite/delegates/gpu/gl_delegate.h"
#endif // !MEDIAPIPE_DISABLE_GPU
#if defined(__APPLE__) && !TARGET_OS_OSX // iOS
#if defined(MEDIAPIPE_IOS)
#import <CoreVideo/CoreVideo.h>
#import <Metal/Metal.h>
#import <MetalKit/MetalKit.h>
@@ -56,17 +55,15 @@ constexpr int kNumCoordsPerBox = 4;
namespace mediapipe {
#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
!defined(__APPLE__)
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
using ::tflite::gpu::gl::CreateReadWriteShaderStorageBuffer;
using ::tflite::gpu::gl::GlShader;
#endif
#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
!defined(__APPLE__)
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
typedef ::tflite::gpu::gl::GlBuffer GpuTensor;
typedef ::tflite::gpu::gl::GlProgram GpuProgram;
#elif defined(__APPLE__) && !TARGET_OS_OSX // iOS
#elif defined(MEDIAPIPE_IOS)
typedef id<MTLBuffer> GpuTensor;
typedef id<MTLComputePipelineState> GpuProgram;
#endif
@@ -183,11 +180,10 @@ class TfLiteTensorsToDetectionsCalculator : public CalculatorBase {
std::vector<Anchor> anchors_;
bool side_packet_anchors_{};
#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
!defined(__APPLE__)
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
mediapipe::GlCalculatorHelper gpu_helper_;
std::unique_ptr<GPUData> gpu_data_;
#elif defined(__APPLE__) && !TARGET_OS_OSX // iOS
#elif defined(MEDIAPIPE_IOS)
MPPMetalHelper* gpu_helper_ = nullptr;
std::unique_ptr<GPUData> gpu_data_;
#endif
@@ -226,10 +222,9 @@ REGISTER_CALCULATOR(TfLiteTensorsToDetectionsCalculator);
}
if (use_gpu) {
#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
!defined(__APPLE__)
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc));
#elif defined(__APPLE__) && !TARGET_OS_OSX // iOS
#elif defined(MEDIAPIPE_IOS)
MP_RETURN_IF_ERROR([MPPMetalHelper updateContract:cc]);
#endif
}
@@ -243,10 +238,9 @@ REGISTER_CALCULATOR(TfLiteTensorsToDetectionsCalculator);
if (cc->Inputs().HasTag("TENSORS_GPU")) {
gpu_input_ = true;
#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
!defined(__APPLE__)
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
MP_RETURN_IF_ERROR(gpu_helper_.Open(cc));
#elif defined(__APPLE__) && !TARGET_OS_OSX // iOS
#elif defined(MEDIAPIPE_IOS)
gpu_helper_ = [[MPPMetalHelper alloc] initWithCalculatorContext:cc];
RET_CHECK(gpu_helper_);
#endif
@@ -406,8 +400,7 @@ REGISTER_CALCULATOR(TfLiteTensorsToDetectionsCalculator);
}
::mediapipe::Status TfLiteTensorsToDetectionsCalculator::ProcessGPU(
CalculatorContext* cc, std::vector<Detection>* output_detections) {
#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
!defined(__APPLE__)
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
const auto& input_tensors =
cc->Inputs().Tag("TENSORS_GPU").Get<std::vector<GpuTensor>>();
RET_CHECK_GE(input_tensors.size(), 2);
@@ -470,7 +463,7 @@ REGISTER_CALCULATOR(TfLiteTensorsToDetectionsCalculator);
return ::mediapipe::OkStatus();
}));
#elif defined(__APPLE__) && !TARGET_OS_OSX // iOS
#elif defined(MEDIAPIPE_IOS)
const auto& input_tensors =
cc->Inputs().Tag("TENSORS_GPU").Get<std::vector<GpuTensor>>();
@@ -569,12 +562,11 @@ REGISTER_CALCULATOR(TfLiteTensorsToDetectionsCalculator);
::mediapipe::Status TfLiteTensorsToDetectionsCalculator::Close(
CalculatorContext* cc) {
#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
!defined(__APPLE__)
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
gpu_helper_.RunInGlContext([this] { gpu_data_.reset(); });
#elif defined(__APPLE__) && !TARGET_OS_OSX // iOS
#elif defined(MEDIAPIPE_IOS)
gpu_data_.reset();
#endif // !MEDIAPIPE_DISABLE_GPU
#endif
return ::mediapipe::OkStatus();
}
@@ -723,8 +715,7 @@ Detection TfLiteTensorsToDetectionsCalculator::ConvertToDetection(
::mediapipe::Status TfLiteTensorsToDetectionsCalculator::GpuInit(
CalculatorContext* cc) {
#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
!defined(__APPLE__)
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext([this]()
-> ::mediapipe::Status {
gpu_data_ = absl::make_unique<GPUData>();
@@ -937,8 +928,7 @@ void main() {
return ::mediapipe::OkStatus();
}));
#elif defined(__APPLE__) && !TARGET_OS_OSX // iOS
// TODO consolidate Metal and OpenGL shaders via vulkan.
#elif defined(MEDIAPIPE_IOS)
gpu_data_ = absl::make_unique<GPUData>();
id<MTLDevice> device = gpu_helper_.mtlDevice;
@@ -1168,7 +1158,7 @@ kernel void scoreKernel(
CHECK_LT(num_classes_, max_wg_size) << "# classes must be <" << max_wg_size;
}
#endif // __ANDROID__ or iOS
#endif // !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
return ::mediapipe::OkStatus();
}
mediapipe/calculators/tflite/tflite_tensors_to_landmarks_calculator.cc
+25 -24
View File
@@ -76,11 +76,11 @@ REGISTER_CALCULATOR(TfLiteTensorsToLandmarksCalculator);
}
if (cc->Outputs().HasTag("LANDMARKS")) {
cc->Outputs().Tag("LANDMARKS").Set<std::vector<Landmark>>();
cc->Outputs().Tag("LANDMARKS").Set<LandmarkList>();
}
if (cc->Outputs().HasTag("NORM_LANDMARKS")) {
cc->Outputs().Tag("NORM_LANDMARKS").Set<std::vector<NormalizedLandmark>>();
cc->Outputs().Tag("NORM_LANDMARKS").Set<NormalizedLandmarkList>();
}
return ::mediapipe::OkStatus();
@@ -127,54 +127,55 @@ REGISTER_CALCULATOR(TfLiteTensorsToLandmarksCalculator);
const float* raw_landmarks = raw_tensor->data.f;
auto output_landmarks = absl::make_unique<std::vector<Landmark>>();
LandmarkList output_landmarks;
for (int ld = 0; ld < num_landmarks_; ++ld) {
const int offset = ld * num_dimensions;
Landmark landmark;
Landmark* landmark = output_landmarks.add_landmark();
if (options_.flip_horizontally()) {
landmark.set_x(options_.input_image_width() - raw_landmarks[offset]);
landmark->set_x(options_.input_image_width() - raw_landmarks[offset]);
} else {
landmark.set_x(raw_landmarks[offset]);
landmark->set_x(raw_landmarks[offset]);
}
if (num_dimensions > 1) {
if (options_.flip_vertically()) {
landmark.set_y(options_.input_image_height() -
raw_landmarks[offset + 1]);
landmark->set_y(options_.input_image_height() -
raw_landmarks[offset + 1]);
} else {
landmark.set_y(raw_landmarks[offset + 1]);
landmark->set_y(raw_landmarks[offset + 1]);
}
}
if (num_dimensions > 2) {
landmark.set_z(raw_landmarks[offset + 2]);
landmark->set_z(raw_landmarks[offset + 2]);
}
output_landmarks->push_back(landmark);
}
// Output normalized landmarks if required.
if (cc->Outputs().HasTag("NORM_LANDMARKS")) {
auto output_norm_landmarks =
absl::make_unique<std::vector<NormalizedLandmark>>();
for (const auto& landmark : *output_landmarks) {
NormalizedLandmark norm_landmark;
norm_landmark.set_x(static_cast<float>(landmark.x()) /
options_.input_image_width());
norm_landmark.set_y(static_cast<float>(landmark.y()) /
options_.input_image_height());
norm_landmark.set_z(landmark.z() / options_.normalize_z());
output_norm_landmarks->push_back(norm_landmark);
NormalizedLandmarkList output_norm_landmarks;
// for (const auto& landmark : output_landmarks) {
for (int i = 0; i < output_landmarks.landmark_size(); ++i) {
const Landmark& landmark = output_landmarks.landmark(i);
NormalizedLandmark* norm_landmark = output_norm_landmarks.add_landmark();
norm_landmark->set_x(static_cast<float>(landmark.x()) /
options_.input_image_width());
norm_landmark->set_y(static_cast<float>(landmark.y()) /
options_.input_image_height());
norm_landmark->set_z(landmark.z() / options_.normalize_z());
}
cc->Outputs()
.Tag("NORM_LANDMARKS")
.Add(output_norm_landmarks.release(), cc->InputTimestamp());
.AddPacket(MakePacket<NormalizedLandmarkList>(output_norm_landmarks)
.At(cc->InputTimestamp()));
}
// Output absolute landmarks.
if (cc->Outputs().HasTag("LANDMARKS")) {
cc->Outputs()
.Tag("LANDMARKS")
.Add(output_landmarks.release(), cc->InputTimestamp());
.AddPacket(MakePacket<LandmarkList>(output_landmarks)
.At(cc->InputTimestamp()));
}
return ::mediapipe::OkStatus();
mediapipe/calculators/tflite/tflite_tensors_to_segmentation_calculator.cc
+13 -26
View File
@@ -28,8 +28,7 @@
#include "mediapipe/util/resource_util.h"
#include "tensorflow/lite/interpreter.h"
#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
!defined(__APPLE__)
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
#include "mediapipe/gpu/gl_calculator_helper.h"
#include "mediapipe/gpu/gl_simple_shaders.h"
#include "mediapipe/gpu/shader_util.h"
@@ -54,8 +53,7 @@ float Clamp(float val, float min, float max) {
namespace mediapipe {
#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
!defined(__APPLE__)
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
using ::tflite::gpu::gl::CopyBuffer;
using ::tflite::gpu::gl::CreateReadWriteRgbaImageTexture;
using ::tflite::gpu::gl::CreateReadWriteShaderStorageBuffer;
@@ -131,8 +129,7 @@ class TfLiteTensorsToSegmentationCalculator : public CalculatorBase {
int tensor_channels_ = 0;
bool use_gpu_ = false;
#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
!defined(__APPLE__)
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
mediapipe::GlCalculatorHelper gpu_helper_;
std::unique_ptr<GlProgram> mask_program_with_prev_;
std::unique_ptr<GlProgram> mask_program_no_prev_;
@@ -162,8 +159,7 @@ REGISTER_CALCULATOR(TfLiteTensorsToSegmentationCalculator);
}
// Inputs GPU.
#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
!defined(__APPLE__)
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
if (cc->Inputs().HasTag("TENSORS_GPU")) {
cc->Inputs().Tag("TENSORS_GPU").Set<std::vector<GlBuffer>>();
use_gpu |= true;
@@ -182,8 +178,7 @@ REGISTER_CALCULATOR(TfLiteTensorsToSegmentationCalculator);
if (cc->Outputs().HasTag("MASK")) {
cc->Outputs().Tag("MASK").Set<ImageFrame>();
}
#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
!defined(__APPLE__)
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
if (cc->Outputs().HasTag("MASK_GPU")) {
cc->Outputs().Tag("MASK_GPU").Set<mediapipe::GpuBuffer>();
use_gpu |= true;
@@ -191,8 +186,7 @@ REGISTER_CALCULATOR(TfLiteTensorsToSegmentationCalculator);
#endif // !MEDIAPIPE_DISABLE_GPU
if (use_gpu) {
#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
!defined(__APPLE__)
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc));
#endif // !MEDIAPIPE_DISABLE_GPU
}
@@ -205,8 +199,7 @@ REGISTER_CALCULATOR(TfLiteTensorsToSegmentationCalculator);
if (cc->Inputs().HasTag("TENSORS_GPU")) {
use_gpu_ = true;
#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
!defined(__APPLE__)
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
MP_RETURN_IF_ERROR(gpu_helper_.Open(cc));
#endif // !MEDIAPIPE_DISABLE_GPU
}
@@ -214,8 +207,7 @@ REGISTER_CALCULATOR(TfLiteTensorsToSegmentationCalculator);
MP_RETURN_IF_ERROR(LoadOptions(cc));
if (use_gpu_) {
#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
!defined(__APPLE__)
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
MP_RETURN_IF_ERROR(
gpu_helper_.RunInGlContext([this, cc]() -> ::mediapipe::Status {
MP_RETURN_IF_ERROR(InitGpu(cc));
@@ -232,8 +224,7 @@ REGISTER_CALCULATOR(TfLiteTensorsToSegmentationCalculator);
::mediapipe::Status TfLiteTensorsToSegmentationCalculator::Process(
CalculatorContext* cc) {
if (use_gpu_) {
#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
!defined(__APPLE__)
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
MP_RETURN_IF_ERROR(
gpu_helper_.RunInGlContext([this, cc]() -> ::mediapipe::Status {
MP_RETURN_IF_ERROR(ProcessGpu(cc));
@@ -249,8 +240,7 @@ REGISTER_CALCULATOR(TfLiteTensorsToSegmentationCalculator);
::mediapipe::Status TfLiteTensorsToSegmentationCalculator::Close(
CalculatorContext* cc) {
#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
!defined(__APPLE__)
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
gpu_helper_.RunInGlContext([this] {
if (upsample_program_) glDeleteProgram(upsample_program_);
upsample_program_ = 0;
@@ -377,8 +367,7 @@ REGISTER_CALCULATOR(TfLiteTensorsToSegmentationCalculator);
if (cc->Inputs().Tag("TENSORS_GPU").IsEmpty()) {
return ::mediapipe::OkStatus();
}
#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
!defined(__APPLE__)
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
// Get input streams.
const auto& input_tensors =
cc->Inputs().Tag("TENSORS_GPU").Get<std::vector<GlBuffer>>();
@@ -464,8 +453,7 @@ REGISTER_CALCULATOR(TfLiteTensorsToSegmentationCalculator);
}
void TfLiteTensorsToSegmentationCalculator::GlRender() {
#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
!defined(__APPLE__)
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
static const GLfloat square_vertices[] = {
-1.0f, -1.0f, // bottom left
1.0f, -1.0f, // bottom right
@@ -537,8 +525,7 @@ void TfLiteTensorsToSegmentationCalculator::GlRender() {
::mediapipe::Status TfLiteTensorsToSegmentationCalculator::InitGpu(
CalculatorContext* cc) {
#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
!defined(__APPLE__)
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext([this]()
-> ::mediapipe::Status {
// A shader to process a segmentation tensor into an output mask,
mediapipe/calculators/util/detection_label_id_to_text_calculator.cc
+1 -2
View File
@@ -19,8 +19,7 @@
#include "mediapipe/framework/port/status.h"
#include "mediapipe/util/resource_util.h"
#if defined(MEDIAPIPE_LITE) || defined(__EMSCRIPTEN__) || \
defined(__ANDROID__) || (defined(__APPLE__) && !TARGET_OS_OSX)
#if defined(MEDIAPIPE_MOBILE)
#include "mediapipe/util/android/file/base/file.h"
#include "mediapipe/util/android/file/base/helpers.h"
#else
mediapipe/calculators/util/filter_collection_calculator.cc
+3 -3
View File
@@ -27,8 +27,8 @@ typedef FilterCollectionCalculator<std::vector<::mediapipe::NormalizedRect>>
REGISTER_CALCULATOR(FilterNormalizedRectCollectionCalculator);
typedef FilterCollectionCalculator<
std::vector<std::vector<::mediapipe::NormalizedLandmark>>>
FilterLandmarksCollectionCalculator;
REGISTER_CALCULATOR(FilterLandmarksCollectionCalculator);
std::vector<::mediapipe::NormalizedLandmarkList>>
FilterLandmarkListCollectionCalculator;
REGISTER_CALCULATOR(FilterLandmarkListCollectionCalculator);
} // namespace mediapipe
mediapipe/calculators/util/landmark_letterbox_removal_calculator.cc
+15 -16
View File
@@ -49,7 +49,7 @@ constexpr char kLetterboxPaddingTag[] = "LETTERBOX_PADDING";
// corresponding input image before letterboxing.
//
// Input:
// LANDMARKS: An std::vector<NormalizedLandmark> representing landmarks on an
// LANDMARKS: A NormalizedLandmarkList representing landmarks on an
// letterboxed image.
//
// LETTERBOX_PADDING: An std::array<float, 4> representing the letterbox
@@ -57,7 +57,7 @@ constexpr char kLetterboxPaddingTag[] = "LETTERBOX_PADDING";
// image, normalized to [0.f, 1.f] by the letterboxed image dimensions.
//
// Output:
// LANDMARKS: An std::vector<NormalizedLandmark> representing landmarks with
// LANDMARKS: An NormalizedLandmarkList proto representing landmarks with
// their locations adjusted to the letterbox-removed (non-padded) image.
//
// Usage example:
@@ -74,10 +74,10 @@ class LandmarkLetterboxRemovalCalculator : public CalculatorBase {
cc->Inputs().HasTag(kLetterboxPaddingTag))
<< "Missing one or more input streams.";
cc->Inputs().Tag(kLandmarksTag).Set<std::vector<NormalizedLandmark>>();
cc->Inputs().Tag(kLandmarksTag).Set<NormalizedLandmarkList>();
cc->Inputs().Tag(kLetterboxPaddingTag).Set<std::array<float, 4>>();
cc->Outputs().Tag(kLandmarksTag).Set<std::vector<NormalizedLandmark>>();
cc->Outputs().Tag(kLandmarksTag).Set<NormalizedLandmarkList>();
return ::mediapipe::OkStatus();
}
@@ -94,8 +94,8 @@ class LandmarkLetterboxRemovalCalculator : public CalculatorBase {
return ::mediapipe::OkStatus();
}
const auto& input_landmarks =
cc->Inputs().Tag(kLandmarksTag).Get<std::vector<NormalizedLandmark>>();
const NormalizedLandmarkList& input_landmarks =
cc->Inputs().Tag(kLandmarksTag).Get<NormalizedLandmarkList>();
const auto& letterbox_padding =
cc->Inputs().Tag(kLetterboxPaddingTag).Get<std::array<float, 4>>();
@@ -104,24 +104,23 @@ class LandmarkLetterboxRemovalCalculator : public CalculatorBase {
const float left_and_right = letterbox_padding[0] + letterbox_padding[2];
const float top_and_bottom = letterbox_padding[1] + letterbox_padding[3];
auto output_landmarks =
absl::make_unique<std::vector<NormalizedLandmark>>();
for (const auto& landmark : input_landmarks) {
NormalizedLandmark new_landmark;
NormalizedLandmarkList output_landmarks;
for (int i = 0; i < input_landmarks.landmark_size(); ++i) {
const NormalizedLandmark& landmark = input_landmarks.landmark(i);
NormalizedLandmark* new_landmark = output_landmarks.add_landmark();
const float new_x = (landmark.x() - left) / (1.0f - left_and_right);
const float new_y = (landmark.y() - top) / (1.0f - top_and_bottom);
new_landmark.set_x(new_x);
new_landmark.set_y(new_y);
new_landmark->set_x(new_x);
new_landmark->set_y(new_y);
// Keep z-coord as is.
new_landmark.set_z(landmark.z());
output_landmarks->emplace_back(new_landmark);
new_landmark->set_z(landmark.z());
}
cc->Outputs()
.Tag(kLandmarksTag)
.Add(output_landmarks.release(), cc->InputTimestamp());
.AddPacket(MakePacket<NormalizedLandmarkList>(output_landmarks)
.At(cc->InputTimestamp()));
return ::mediapipe::OkStatus();
}
};
mediapipe/calculators/util/landmark_letterbox_removal_calculator_test.cc
+27 -26
View File
@@ -43,10 +43,10 @@ CalculatorGraphConfig::Node GetDefaultNode() {
TEST(LandmarkLetterboxRemovalCalculatorTest, PaddingLeftRight) {
CalculatorRunner runner(GetDefaultNode());
auto landmarks = absl::make_unique<std::vector<NormalizedLandmark>>();
landmarks->push_back(CreateLandmark(0.5f, 0.5f));
landmarks->push_back(CreateLandmark(0.2f, 0.2f));
landmarks->push_back(CreateLandmark(0.7f, 0.7f));
auto landmarks = absl::make_unique<NormalizedLandmarkList>();
*landmarks->add_landmark() = CreateLandmark(0.5f, 0.5f);
*landmarks->add_landmark() = CreateLandmark(0.2f, 0.2f);
*landmarks->add_landmark() = CreateLandmark(0.7f, 0.7f);
runner.MutableInputs()
->Tag("LANDMARKS")
.packets.push_back(
@@ -61,26 +61,28 @@ TEST(LandmarkLetterboxRemovalCalculatorTest, PaddingLeftRight) {
MP_ASSERT_OK(runner.Run()) << "Calculator execution failed.";
const std::vector<Packet>& output = runner.Outputs().Tag("LANDMARKS").packets;
ASSERT_EQ(1, output.size());
const auto& output_landmarks =
output[0].Get<std::vector<NormalizedLandmark>>();
const auto& output_landmarks = output[0].Get<NormalizedLandmarkList>();
EXPECT_EQ(output_landmarks.size(), 3);
EXPECT_EQ(output_landmarks.landmark_size(), 3);
EXPECT_THAT(output_landmarks[0].x(), testing::FloatNear(0.6f, 1e-5));
EXPECT_THAT(output_landmarks[0].y(), testing::FloatNear(0.5f, 1e-5));
EXPECT_THAT(output_landmarks[1].x(), testing::FloatNear(0.0f, 1e-5));
EXPECT_THAT(output_landmarks[1].y(), testing::FloatNear(0.2f, 1e-5));
EXPECT_THAT(output_landmarks[2].x(), testing::FloatNear(1.0f, 1e-5));
EXPECT_THAT(output_landmarks[2].y(), testing::FloatNear(0.7f, 1e-5));
EXPECT_THAT(output_landmarks.landmark(0).x(), testing::FloatNear(0.6f, 1e-5));
EXPECT_THAT(output_landmarks.landmark(0).y(), testing::FloatNear(0.5f, 1e-5));
EXPECT_THAT(output_landmarks.landmark(1).x(), testing::FloatNear(0.0f, 1e-5));
EXPECT_THAT(output_landmarks.landmark(1).y(), testing::FloatNear(0.2f, 1e-5));
EXPECT_THAT(output_landmarks.landmark(2).x(), testing::FloatNear(1.0f, 1e-5));
EXPECT_THAT(output_landmarks.landmark(2).y(), testing::FloatNear(0.7f, 1e-5));
}
TEST(LandmarkLetterboxRemovalCalculatorTest, PaddingTopBottom) {
CalculatorRunner runner(GetDefaultNode());
auto landmarks = absl::make_unique<std::vector<NormalizedLandmark>>();
landmarks->push_back(CreateLandmark(0.5f, 0.5f));
landmarks->push_back(CreateLandmark(0.2f, 0.2f));
landmarks->push_back(CreateLandmark(0.7f, 0.7f));
auto landmarks = absl::make_unique<NormalizedLandmarkList>();
NormalizedLandmark* landmark = landmarks->add_landmark();
*landmark = CreateLandmark(0.5f, 0.5f);
landmark = landmarks->add_landmark();
*landmark = CreateLandmark(0.2f, 0.2f);
landmark = landmarks->add_landmark();
*landmark = CreateLandmark(0.7f, 0.7f);
runner.MutableInputs()
->Tag("LANDMARKS")
.packets.push_back(
@@ -95,17 +97,16 @@ TEST(LandmarkLetterboxRemovalCalculatorTest, PaddingTopBottom) {
MP_ASSERT_OK(runner.Run()) << "Calculator execution failed.";
const std::vector<Packet>& output = runner.Outputs().Tag("LANDMARKS").packets;
ASSERT_EQ(1, output.size());
const auto& output_landmarks =
output[0].Get<std::vector<NormalizedLandmark>>();
const auto& output_landmarks = output[0].Get<NormalizedLandmarkList>();
EXPECT_EQ(output_landmarks.size(), 3);
EXPECT_EQ(output_landmarks.landmark_size(), 3);
EXPECT_THAT(output_landmarks[0].x(), testing::FloatNear(0.5f, 1e-5));
EXPECT_THAT(output_landmarks[0].y(), testing::FloatNear(0.6f, 1e-5));
EXPECT_THAT(output_landmarks[1].x(), testing::FloatNear(0.2f, 1e-5));
EXPECT_THAT(output_landmarks[1].y(), testing::FloatNear(0.0f, 1e-5));
EXPECT_THAT(output_landmarks[2].x(), testing::FloatNear(0.7f, 1e-5));
EXPECT_THAT(output_landmarks[2].y(), testing::FloatNear(1.0f, 1e-5));
EXPECT_THAT(output_landmarks.landmark(0).x(), testing::FloatNear(0.5f, 1e-5));
EXPECT_THAT(output_landmarks.landmark(0).y(), testing::FloatNear(0.6f, 1e-5));
EXPECT_THAT(output_landmarks.landmark(1).x(), testing::FloatNear(0.2f, 1e-5));
EXPECT_THAT(output_landmarks.landmark(1).y(), testing::FloatNear(0.0f, 1e-5));
EXPECT_THAT(output_landmarks.landmark(2).x(), testing::FloatNear(0.7f, 1e-5));
EXPECT_THAT(output_landmarks.landmark(2).y(), testing::FloatNear(1.0f, 1e-5));
}
} // namespace mediapipe
mediapipe/calculators/util/landmark_projection_calculator.cc
+15 -16
View File
@@ -47,13 +47,13 @@ constexpr char kRectTag[] = "NORM_RECT";
// Projects normalized landmarks in a rectangle to its original coordinates. The
// rectangle must also be in normalized coordinates.
// Input:
// NORM_LANDMARKS: An std::vector<NormalizedLandmark> representing landmarks
// NORM_LANDMARKS: A NormalizedLandmarkList representing landmarks
// in a normalized rectangle.
// NORM_RECT: An NormalizedRect representing a normalized rectangle in image
// coordinates.
//
// Output:
// NORM_LANDMARKS: An std::vector<NormalizedLandmark> representing landmarks
// NORM_LANDMARKS: A NormalizedLandmarkList representing landmarks
// with their locations adjusted to the image.
//
// Usage example:
@@ -70,10 +70,10 @@ class LandmarkProjectionCalculator : public CalculatorBase {
cc->Inputs().HasTag(kRectTag))
<< "Missing one or more input streams.";
cc->Inputs().Tag(kLandmarksTag).Set<std::vector<NormalizedLandmark>>();
cc->Inputs().Tag(kLandmarksTag).Set<NormalizedLandmarkList>();
cc->Inputs().Tag(kRectTag).Set<NormalizedRect>();
cc->Outputs().Tag(kLandmarksTag).Set<std::vector<NormalizedLandmark>>();
cc->Outputs().Tag(kLandmarksTag).Set<NormalizedLandmarkList>();
return ::mediapipe::OkStatus();
}
@@ -92,14 +92,14 @@ class LandmarkProjectionCalculator : public CalculatorBase {
return ::mediapipe::OkStatus();
}
const auto& input_landmarks =
cc->Inputs().Tag(kLandmarksTag).Get<std::vector<NormalizedLandmark>>();
const NormalizedLandmarkList& input_landmarks =
cc->Inputs().Tag(kLandmarksTag).Get<NormalizedLandmarkList>();
const auto& input_rect = cc->Inputs().Tag(kRectTag).Get<NormalizedRect>();
auto output_landmarks =
absl::make_unique<std::vector<NormalizedLandmark>>();
for (const auto& landmark : input_landmarks) {
NormalizedLandmark new_landmark;
NormalizedLandmarkList output_landmarks;
for (int i = 0; i < input_landmarks.landmark_size(); ++i) {
const NormalizedLandmark& landmark = input_landmarks.landmark(i);
NormalizedLandmark* new_landmark = output_landmarks.add_landmark();
const float x = landmark.x() - 0.5f;
const float y = landmark.y() - 0.5f;
@@ -110,17 +110,16 @@ class LandmarkProjectionCalculator : public CalculatorBase {
new_x = new_x * input_rect.width() + input_rect.x_center();
new_y = new_y * input_rect.height() + input_rect.y_center();
new_landmark.set_x(new_x);
new_landmark.set_y(new_y);
new_landmark->set_x(new_x);
new_landmark->set_y(new_y);
// Keep z-coord as is.
new_landmark.set_z(landmark.z());
output_landmarks->emplace_back(new_landmark);
new_landmark->set_z(landmark.z());
}
cc->Outputs()
.Tag(kLandmarksTag)
.Add(output_landmarks.release(), cc->InputTimestamp());
.AddPacket(MakePacket<NormalizedLandmarkList>(output_landmarks)
.At(cc->InputTimestamp()));
return ::mediapipe::OkStatus();
}
};
mediapipe/calculators/util/landmarks_to_detection_calculator.cc
+15 -16
View File
@@ -28,8 +28,7 @@ namespace {
constexpr char kDetectionTag[] = "DETECTION";
constexpr char kNormalizedLandmarksTag[] = "NORM_LANDMARKS";
Detection ConvertLandmarksToDetection(
const std::vector<NormalizedLandmark>& landmarks) {
Detection ConvertLandmarksToDetection(const NormalizedLandmarkList& landmarks) {
Detection detection;
LocationData* location_data = detection.mutable_location_data();
@@ -37,7 +36,8 @@ Detection ConvertLandmarksToDetection(
float x_max = std::numeric_limits<float>::min();
float y_min = std::numeric_limits<float>::max();
float y_max = std::numeric_limits<float>::min();
for (const auto& landmark : landmarks) {
for (int i = 0; i < landmarks.landmark_size(); ++i) {
const NormalizedLandmark& landmark = landmarks.landmark(i);
x_min = std::min(x_min, landmark.x());
x_max = std::max(x_max, landmark.x());
y_min = std::min(y_min, landmark.y());
@@ -67,7 +67,7 @@ Detection ConvertLandmarksToDetection(
// to specify a subset of landmarks for creating the detection.
//
// Input:
// NOMR_LANDMARKS: A vector of NormalizedLandmark.
// NOMR_LANDMARKS: A NormalizedLandmarkList proto.
//
// Output:
// DETECTION: A Detection proto.
@@ -95,9 +95,7 @@ REGISTER_CALCULATOR(LandmarksToDetectionCalculator);
RET_CHECK(cc->Inputs().HasTag(kNormalizedLandmarksTag));
RET_CHECK(cc->Outputs().HasTag(kDetectionTag));
// TODO: Also support converting Landmark to Detection.
cc->Inputs()
.Tag(kNormalizedLandmarksTag)
.Set<std::vector<NormalizedLandmark>>();
cc->Inputs().Tag(kNormalizedLandmarksTag).Set<NormalizedLandmarkList>();
cc->Outputs().Tag(kDetectionTag).Set<Detection>();
return ::mediapipe::OkStatus();
@@ -113,19 +111,20 @@ REGISTER_CALCULATOR(LandmarksToDetectionCalculator);
::mediapipe::Status LandmarksToDetectionCalculator::Process(
CalculatorContext* cc) {
const auto& landmarks = cc->Inputs()
.Tag(kNormalizedLandmarksTag)
.Get<std::vector<NormalizedLandmark>>();
RET_CHECK_GT(landmarks.size(), 0) << "Input landmark vector is empty.";
const auto& landmarks =
cc->Inputs().Tag(kNormalizedLandmarksTag).Get<NormalizedLandmarkList>();
RET_CHECK_GT(landmarks.landmark_size(), 0)
<< "Input landmark vector is empty.";
auto detection = absl::make_unique<Detection>();
if (options_.selected_landmark_indices_size()) {
std::vector<NormalizedLandmark> subset_landmarks(
options_.selected_landmark_indices_size());
for (int i = 0; i < subset_landmarks.size(); ++i) {
RET_CHECK_LT(options_.selected_landmark_indices(i), landmarks.size())
NormalizedLandmarkList subset_landmarks;
for (int i = 0; i < options_.selected_landmark_indices_size(); ++i) {
RET_CHECK_LT(options_.selected_landmark_indices(i),
landmarks.landmark_size())
<< "Index of landmark subset is out of range.";
subset_landmarks[i] = landmarks[options_.selected_landmark_indices(i)];
*subset_landmarks.add_landmark() =
landmarks.landmark(options_.selected_landmark_indices(i));
}
*detection = ConvertLandmarksToDetection(subset_landmarks);
} else {
mediapipe/calculators/util/landmarks_to_floats_calculator.cc
+10 -9
View File
@@ -48,7 +48,7 @@ constexpr char kMatrixTag[] = "MATRIX";
// Converts a vector of landmarks to a vector of floats or a matrix.
// Input:
// NORM_LANDMARKS: An std::vector<NormalizedLandmark>.
// NORM_LANDMARKS: A NormalizedLandmarkList proto.
//
// Output:
// FLOATS(optional): A vector of floats from flattened landmarks.
@@ -63,7 +63,7 @@ constexpr char kMatrixTag[] = "MATRIX";
class LandmarksToFloatsCalculator : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
cc->Inputs().Tag(kLandmarksTag).Set<std::vector<NormalizedLandmark>>();
cc->Inputs().Tag(kLandmarksTag).Set<NormalizedLandmarkList>();
RET_CHECK(cc->Outputs().HasTag(kFloatsTag) ||
cc->Outputs().HasTag(kMatrixTag));
if (cc->Outputs().HasTag(kFloatsTag)) {
@@ -94,11 +94,12 @@ class LandmarksToFloatsCalculator : public CalculatorBase {
}
const auto& input_landmarks =
cc->Inputs().Tag(kLandmarksTag).Get<std::vector<NormalizedLandmark>>();
cc->Inputs().Tag(kLandmarksTag).Get<NormalizedLandmarkList>();
if (cc->Outputs().HasTag(kFloatsTag)) {
auto output_floats = absl::make_unique<std::vector<float>>();
for (const auto& landmark : input_landmarks) {
for (int i = 0; i < input_landmarks.landmark_size(); ++i) {
const NormalizedLandmark& landmark = input_landmarks.landmark(i);
output_floats->emplace_back(landmark.x());
if (num_dimensions_ > 1) {
output_floats->emplace_back(landmark.y());
@@ -113,14 +114,14 @@ class LandmarksToFloatsCalculator : public CalculatorBase {
.Add(output_floats.release(), cc->InputTimestamp());
} else {
auto output_matrix = absl::make_unique<Matrix>();
output_matrix->setZero(num_dimensions_, input_landmarks.size());
for (int i = 0; i < input_landmarks.size(); ++i) {
(*output_matrix)(0, i) = input_landmarks[i].x();
output_matrix->setZero(num_dimensions_, input_landmarks.landmark_size());
for (int i = 0; i < input_landmarks.landmark_size(); ++i) {
(*output_matrix)(0, i) = input_landmarks.landmark(i).x();
if (num_dimensions_ > 1) {
(*output_matrix)(1, i) = input_landmarks[i].y();
(*output_matrix)(1, i) = input_landmarks.landmark(i).y();
}
if (num_dimensions_ > 2) {
(*output_matrix)(2, i) = input_landmarks[i].z();
(*output_matrix)(2, i) = input_landmarks.landmark(i).z();
}
}
cc->Outputs()
mediapipe/calculators/util/landmarks_to_render_data_calculator.cc
+39 -34
View File
@@ -46,12 +46,13 @@ inline float Remap(float x, float lo, float hi, float scale) {
return (x - lo) / (hi - lo + 1e-6) * scale;
}
template <class LandmarkType>
inline void GetMinMaxZ(const std::vector<LandmarkType>& landmarks, float* z_min,
template <class LandmarkListType, class LandmarkType>
inline void GetMinMaxZ(const LandmarkListType& landmarks, float* z_min,
float* z_max) {
*z_min = std::numeric_limits<float>::max();
*z_max = std::numeric_limits<float>::min();
for (const auto& landmark : landmarks) {
for (int i = 0; i < landmarks.landmark_size(); ++i) {
const LandmarkType& landmark = landmarks.landmark(i);
*z_min = std::min(landmark.z(), *z_min);
*z_max = std::max(landmark.z(), *z_max);
}
@@ -73,7 +74,7 @@ void SetColorSizeValueFromZ(float z, float z_min, float z_max,
} // namespace
// A calculator that converts Landmark proto to RenderData proto for
// visualization. The input should be std::vector<Landmark>. It is also possible
// visualization. The input should be LandmarkList proto. It is also possible
// to specify the connections between landmarks.
//
// Example config:
@@ -121,11 +122,11 @@ class LandmarksToRenderDataCalculator : public CalculatorBase {
const LandmarksToRenderDataCalculatorOptions& options, bool normalized,
int gray_val1, int gray_val2, RenderData* render_data);
template <class LandmarkType>
void AddConnections(const std::vector<LandmarkType>& landmarks,
bool normalized, RenderData* render_data);
template <class LandmarkType>
void AddConnectionsWithDepth(const std::vector<LandmarkType>& landmarks,
template <class LandmarkListType>
void AddConnections(const LandmarkListType& landmarks, bool normalized,
RenderData* render_data);
template <class LandmarkListType>
void AddConnectionsWithDepth(const LandmarkListType& landmarks,
bool normalized, float min_z, float max_z,
RenderData* render_data);
@@ -144,10 +145,10 @@ REGISTER_CALCULATOR(LandmarksToRenderDataCalculator);
"normalized landmarks.";
if (cc->Inputs().HasTag(kLandmarksTag)) {
cc->Inputs().Tag(kLandmarksTag).Set<std::vector<Landmark>>();
cc->Inputs().Tag(kLandmarksTag).Set<LandmarkList>();
}
if (cc->Inputs().HasTag(kNormLandmarksTag)) {
cc->Inputs().Tag(kNormLandmarksTag).Set<std::vector<NormalizedLandmark>>();
cc->Inputs().Tag(kNormLandmarksTag).Set<NormalizedLandmarkList>();
}
cc->Outputs().Tag(kRenderDataTag).Set<RenderData>();
return ::mediapipe::OkStatus();
@@ -169,16 +170,17 @@ REGISTER_CALCULATOR(LandmarksToRenderDataCalculator);
float z_max = 0.f;
if (cc->Inputs().HasTag(kLandmarksTag)) {
const auto& landmarks =
cc->Inputs().Tag(kLandmarksTag).Get<std::vector<Landmark>>();
const LandmarkList& landmarks =
cc->Inputs().Tag(kLandmarksTag).Get<LandmarkList>();
RET_CHECK_EQ(options_.landmark_connections_size() % 2, 0)
<< "Number of entries in landmark connections must be a multiple of 2";
if (visualize_depth) {
GetMinMaxZ(landmarks, &z_min, &z_max);
GetMinMaxZ<LandmarkList, Landmark>(landmarks, &z_min, &z_max);
}
// Only change rendering if there are actually z values other than 0.
visualize_depth &= ((z_max - z_min) > 1e-3);
for (const auto& landmark : landmarks) {
for (int i = 0; i < landmarks.landmark_size(); ++i) {
const Landmark& landmark = landmarks.landmark(i);
auto* landmark_data_render =
AddPointRenderData(options_, render_data.get());
if (visualize_depth) {
@@ -191,25 +193,27 @@ REGISTER_CALCULATOR(LandmarksToRenderDataCalculator);
landmark_data->set_y(landmark.y());
}
if (visualize_depth) {
AddConnectionsWithDepth(landmarks, /*normalized=*/false, z_min, z_max,
render_data.get());
AddConnectionsWithDepth<LandmarkList>(landmarks, /*normalized=*/false,
z_min, z_max, render_data.get());
} else {
AddConnections(landmarks, /*normalized=*/false, render_data.get());
AddConnections<LandmarkList>(landmarks, /*normalized=*/false,
render_data.get());
}
}
if (cc->Inputs().HasTag(kNormLandmarksTag)) {
const auto& landmarks = cc->Inputs()
.Tag(kNormLandmarksTag)
.Get<std::vector<NormalizedLandmark>>();
const NormalizedLandmarkList& landmarks =
cc->Inputs().Tag(kNormLandmarksTag).Get<NormalizedLandmarkList>();
RET_CHECK_EQ(options_.landmark_connections_size() % 2, 0)
<< "Number of entries in landmark connections must be a multiple of 2";
if (visualize_depth) {
GetMinMaxZ(landmarks, &z_min, &z_max);
GetMinMaxZ<NormalizedLandmarkList, NormalizedLandmark>(landmarks, &z_min,
&z_max);
}
// Only change rendering if there are actually z values other than 0.
visualize_depth &= ((z_max - z_min) > 1e-3);
for (const auto& landmark : landmarks) {
for (int i = 0; i < landmarks.landmark_size(); ++i) {
const NormalizedLandmark& landmark = landmarks.landmark(i);
auto* landmark_data_render =
AddPointRenderData(options_, render_data.get());
if (visualize_depth) {
@@ -222,10 +226,11 @@ REGISTER_CALCULATOR(LandmarksToRenderDataCalculator);
landmark_data->set_y(landmark.y());
}
if (visualize_depth) {
AddConnectionsWithDepth(landmarks, /*normalized=*/true, z_min, z_max,
render_data.get());
AddConnectionsWithDepth<NormalizedLandmarkList>(
landmarks, /*normalized=*/true, z_min, z_max, render_data.get());
} else {
AddConnections(landmarks, /*normalized=*/true, render_data.get());
AddConnections<NormalizedLandmarkList>(landmarks, /*normalized=*/true,
render_data.get());
}
}
@@ -235,13 +240,13 @@ REGISTER_CALCULATOR(LandmarksToRenderDataCalculator);
return ::mediapipe::OkStatus();
}
template <class LandmarkType>
template <class LandmarkListType>
void LandmarksToRenderDataCalculator::AddConnectionsWithDepth(
const std::vector<LandmarkType>& landmarks, bool normalized, float min_z,
const LandmarkListType& landmarks, bool normalized, float min_z,
float max_z, RenderData* render_data) {
for (int i = 0; i < options_.landmark_connections_size(); i += 2) {
const auto& ld0 = landmarks[options_.landmark_connections(i)];
const auto& ld1 = landmarks[options_.landmark_connections(i + 1)];
const auto& ld0 = landmarks.landmark(options_.landmark_connections(i));
const auto& ld1 = landmarks.landmark(options_.landmark_connections(i + 1));
const int gray_val1 =
255 - static_cast<int>(Remap(ld0.z(), min_z, max_z, 255));
const int gray_val2 =
@@ -272,13 +277,13 @@ void LandmarksToRenderDataCalculator::AddConnectionToRenderData(
connection_annotation->set_thickness(options.thickness());
}
template <class LandmarkType>
template <class LandmarkListType>
void LandmarksToRenderDataCalculator::AddConnections(
const std::vector<LandmarkType>& landmarks, bool normalized,
const LandmarkListType& landmarks, bool normalized,
RenderData* render_data) {
for (int i = 0; i < options_.landmark_connections_size(); i += 2) {
const auto& ld0 = landmarks[options_.landmark_connections(i)];
const auto& ld1 = landmarks[options_.landmark_connections(i + 1)];
const auto& ld0 = landmarks.landmark(options_.landmark_connections(i));
const auto& ld1 = landmarks.landmark(options_.landmark_connections(i + 1));
AddConnectionToRenderData(ld0.x(), ld0.y(), ld1.x(), ld1.y(), options_,
normalized, render_data);
}
mediapipe/calculators/util/top_k_scores_calculator.cc
+1 -2
View File
@@ -29,8 +29,7 @@
#include "mediapipe/framework/port/statusor.h"
#include "mediapipe/util/resource_util.h"
#if defined(MEDIAPIPE_LITE) || defined(__EMSCRIPTEN__) || \
defined(__ANDROID__) || (defined(__APPLE__) && !TARGET_OS_OSX)
#if defined(MEDIAPIPE_MOBILE)
#include "mediapipe/util/android/file/base/file.h"
#include "mediapipe/util/android/file/base/helpers.h"
#else
mediapipe/calculators/video/opencv_video_decoder_calculator.cc
+38
View File
@@ -12,6 +12,8 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include <stdlib.h>
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/image_format.pb.h"
#include "mediapipe/framework/formats/image_frame.h"
@@ -66,6 +68,20 @@ ImageFormat::Format GetImageFormat(int num_channels) {
// output_stream: "VIDEO:video_frames"
// output_stream: "VIDEO_PRESTREAM:video_header"
// }
//
// OpenCV's VideoCapture doesn't decode audio tracks. If the audio tracks need
// to be saved, specify an output side packet with tag "SAVED_AUDIO_PATH".
// The calculator will call FFmpeg binary to save audio tracks as an aac file.
//
// Example config:
// node {
// calculator: "OpenCvVideoDecoderCalculator"
// input_side_packet: "INPUT_FILE_PATH:input_file_path"
// output_side_packet: "SAVED_AUDIO_PATH:audio_path"
// output_stream: "VIDEO:video_frames"
// output_stream: "VIDEO_PRESTREAM:video_header"
// }
//
class OpenCvVideoDecoderCalculator : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
@@ -74,6 +90,9 @@ class OpenCvVideoDecoderCalculator : public CalculatorBase {
if (cc->Outputs().HasTag("VIDEO_PRESTREAM")) {
cc->Outputs().Tag("VIDEO_PRESTREAM").Set<VideoHeader>();
}
if (cc->OutputSidePackets().HasTag("SAVED_AUDIO_PATH")) {
cc->OutputSidePackets().Tag("SAVED_AUDIO_PATH").Set<std::string>();
}
return ::mediapipe::OkStatus();
}
@@ -127,6 +146,25 @@ class OpenCvVideoDecoderCalculator : public CalculatorBase {
}
// Rewind to the very first frame.
cap_->set(cv::CAP_PROP_POS_AVI_RATIO, 0);
if (cc->OutputSidePackets().HasTag("SAVED_AUDIO_PATH")) {
#ifdef HAVE_FFMPEG
std::string saved_audio_path = std::tmpnam(nullptr);
system(absl::StrCat("ffmpeg -nostats -loglevel 0 -i ", input_file_path,
" -vn -f adts ", saved_audio_path)
.c_str());
cc->OutputSidePackets()
.Tag("SAVED_AUDIO_PATH")
.Set(MakePacket<std::string>(saved_audio_path));
#else
return ::mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
<< "OpenCVVideoDecoderCalculator can't save the audio file "
"because FFmpeg is not installed. Please remove "
"output_side_packet: \"SAVED_AUDIO_PATH\" from the node "
"config.";
#endif
}
return ::mediapipe::OkStatus();
}
mediapipe/calculators/video/opencv_video_encoder_calculator.cc
+47 -2
View File
@@ -12,6 +12,8 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include <stdlib.h>
#include <memory>
#include <string>
#include <vector>
@@ -39,8 +41,7 @@ namespace mediapipe {
// packet. Currently, the calculator only supports one video stream (in
// mediapipe::ImageFrame).
//
// Example config to generate the output video file:
//
// Example config:
// node {
// calculator: "OpenCvVideoEncoderCalculator"
// input_stream: "VIDEO:video"
@@ -53,6 +54,26 @@ namespace mediapipe {
// }
// }
// }
//
// OpenCV's VideoWriter doesn't encode audio. If an input side packet with tag
// "AUDIO_FILE_PATH" is specified, the calculator will call FFmpeg binary to
// attach the audio file to the video as the last step in Close().
//
// Example config:
// node {
// calculator: "OpenCvVideoEncoderCalculator"
// input_stream: "VIDEO:video"
// input_stream: "VIDEO_PRESTREAM:video_header"
// input_side_packet: "OUTPUT_FILE_PATH:output_file_path"
// input_side_packet: "AUDIO_FILE_PATH:audio_path"
// node_options {
// [type.googleapis.com/mediapipe.OpenCvVideoEncoderCalculatorOptions]: {
// codec: "avc1"
// video_format: "mp4"
// }
// }
// }
//
class OpenCvVideoEncoderCalculator : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc);
@@ -77,6 +98,9 @@ class OpenCvVideoEncoderCalculator : public CalculatorBase {
}
RET_CHECK(cc->InputSidePackets().HasTag("OUTPUT_FILE_PATH"));
cc->InputSidePackets().Tag("OUTPUT_FILE_PATH").Set<std::string>();
if (cc->InputSidePackets().HasTag("AUDIO_FILE_PATH")) {
cc->InputSidePackets().Tag("AUDIO_FILE_PATH").Set<std::string>();
}
return ::mediapipe::OkStatus();
}
@@ -155,6 +179,27 @@ class OpenCvVideoEncoderCalculator : public CalculatorBase {
if (writer_ && writer_->isOpened()) {
writer_->release();
}
if (cc->InputSidePackets().HasTag("AUDIO_FILE_PATH")) {
#ifdef HAVE_FFMPEG
const std::string& audio_file_path =
cc->InputSidePackets().Tag("AUDIO_FILE_PATH").Get<std::string>();
// A temp output file is needed because FFmpeg can't do in-place editing.
const std::string temp_file_path = std::tmpnam(nullptr);
system(absl::StrCat("mv ", output_file_path_, " ", temp_file_path,
"&& ffmpeg -nostats -loglevel 0 -i ", temp_file_path,
" -i ", audio_file_path,
" -c copy -map 0:v:0 -map 1:a:0 ", output_file_path_,
"&& rm ", temp_file_path)
.c_str());
#else
return ::mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
<< "OpenCVVideoEncoderCalculator can't attach the audio tracks to "
"the video because FFmpeg is not installed. Please remove "
"input_side_packet: \"AUDIO_FILE_PATH\" from the node "
"config.";
#endif
}
return ::mediapipe::OkStatus();
}