talksik/mediapipe
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Project import generated by Copybara.

Browse Source 
GitOrigin-RevId: afeb9cf5a8c069c0a566d16e1622bbb086170e4d
This commit is contained in:
MediaPipe Team
2020-05-21 13:37:51 -04:00
committed by chuoling
parent b6e680647c
commit b133b0f200
258 changed files with 4146 additions and 5147 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mediapipe/calculators/core/BUILD
+1
View File
@@ -500,6 +500,7 @@ cc_library(
"//mediapipe/framework/port:integral_types",
"//mediapipe/framework/port:logging",
"//mediapipe/framework/port:status",
"//mediapipe/framework/tool:options_util",
],
alwayslink = 1,
)
mediapipe/calculators/core/packet_thinner_calculator.cc
+14 -1
View File
@@ -24,11 +24,13 @@
#include "mediapipe/framework/port/integral_types.h"
#include "mediapipe/framework/port/logging.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/framework/tool/options_util.h"
namespace mediapipe {
namespace {
const double kTimebaseUs = 1000000; // Microseconds.
const char* const kOptionsTag = "OPTIONS";
const char* const kPeriodTag = "PERIOD";
} // namespace
@@ -63,9 +65,15 @@ const char* const kPeriodTag = "PERIOD";
// Thinning period can be provided in the calculator options or via a
// side packet with the tag "PERIOD".
//
// Calculator options provided optionally with the "OPTIONS" input
// sidepacket tag will be merged with this calculator's node options, i.e.,
// singular fields of the side packet will overwrite the options defined in the
// node, and repeated fields will concatenate.
//
// Example config:
// node {
// calculator: "PacketThinnerCalculator"
// input_side_packet: "OPTIONS:calculator_options"
// input_stream: "signal"
// output_stream: "output"
// options {
@@ -83,6 +91,9 @@ class PacketThinnerCalculator : public CalculatorBase {
~PacketThinnerCalculator() override {}
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
if (cc->InputSidePackets().HasTag(kOptionsTag)) {
cc->InputSidePackets().Tag(kOptionsTag).Set<CalculatorOptions>();
}
cc->Inputs().Index(0).SetAny();
cc->Outputs().Index(0).SetSameAs(&cc->Inputs().Index(0));
if (cc->InputSidePackets().HasTag(kPeriodTag)) {
@@ -143,7 +154,9 @@ TimestampDiff abs(TimestampDiff t) { return t < 0 ? -t : t; }
} // namespace
::mediapipe::Status PacketThinnerCalculator::Open(CalculatorContext* cc) {
auto& options = cc->Options<PacketThinnerCalculatorOptions>();
PacketThinnerCalculatorOptions options = mediapipe::tool::RetrieveOptions(
cc->Options<PacketThinnerCalculatorOptions>(), cc->InputSidePackets(),
kOptionsTag);
thinner_type_ = options.thinner_type();
// This check enables us to assume only two thinner types exist in Process()
mediapipe/calculators/core/previous_loopback_calculator.cc
+1 -2
View File
@@ -93,8 +93,7 @@ class PreviousLoopbackCalculator : public CalculatorBase {
// MAIN packet, hence not caring about corresponding loop packet.
loop_timestamp = Timestamp::Unset();
}
main_packet_specs_.push_back({.timestamp = main_packet.Timestamp(),
.loop_timestamp = loop_timestamp});
main_packet_specs_.push_back({main_packet.Timestamp(), loop_timestamp});
prev_main_ts_ = main_packet.Timestamp();
}
mediapipe/calculators/image/color_convert_calculator.cc
+24
View File
@@ -38,9 +38,11 @@ void SetColorChannel(int channel, uint8 value, cv::Mat* mat) {
constexpr char kRgbaInTag[] = "RGBA_IN";
constexpr char kRgbInTag[] = "RGB_IN";
constexpr char kBgraInTag[] = "BGRA_IN";
constexpr char kGrayInTag[] = "GRAY_IN";
constexpr char kRgbaOutTag[] = "RGBA_OUT";
constexpr char kRgbOutTag[] = "RGB_OUT";
constexpr char kBgraOutTag[] = "BGRA_OUT";
constexpr char kGrayOutTag[] = "GRAY_OUT";
} // namespace
@@ -53,6 +55,8 @@ constexpr char kGrayOutTag[] = "GRAY_OUT";
// GRAY -> RGB
// RGB -> GRAY
// RGB -> RGBA
// RGBA -> BGRA
// BGRA -> RGBA
//
// This calculator only supports a single input stream and output stream at a
// time. If more than one input stream or output stream is present, the
@@ -63,11 +67,13 @@ constexpr char kGrayOutTag[] = "GRAY_OUT";
// Input streams:
// RGBA_IN: The input video stream (ImageFrame, SRGBA).
// RGB_IN: The input video stream (ImageFrame, SRGB).
// BGRA_IN: The input video stream (ImageFrame, SBGRA).
// GRAY_IN: The input video stream (ImageFrame, GRAY8).
//
// Output streams:
// RGBA_OUT: The output video stream (ImageFrame, SRGBA).
// RGB_OUT: The output video stream (ImageFrame, SRGB).
// BGRA_OUT: The output video stream (ImageFrame, SBGRA).
// GRAY_OUT: The output video stream (ImageFrame, GRAY8).
class ColorConvertCalculator : public CalculatorBase {
public:
@@ -113,6 +119,10 @@ REGISTER_CALCULATOR(ColorConvertCalculator);
cc->Inputs().Tag(kRgbInTag).Set<ImageFrame>();
}
if (cc->Inputs().HasTag(kBgraInTag)) {
cc->Inputs().Tag(kBgraInTag).Set<ImageFrame>();
}
if (cc->Outputs().HasTag(kRgbOutTag)) {
cc->Outputs().Tag(kRgbOutTag).Set<ImageFrame>();
}
@@ -125,6 +135,10 @@ REGISTER_CALCULATOR(ColorConvertCalculator);
cc->Outputs().Tag(kRgbaOutTag).Set<ImageFrame>();
}
if (cc->Outputs().HasTag(kBgraOutTag)) {
cc->Outputs().Tag(kBgraOutTag).Set<ImageFrame>();
}
return ::mediapipe::OkStatus();
}
@@ -171,6 +185,16 @@ REGISTER_CALCULATOR(ColorConvertCalculator);
return ConvertAndOutput(kRgbInTag, kRgbaOutTag, ImageFormat::SRGBA,
cv::COLOR_RGB2RGBA, cc);
}
// BGRA -> RGBA
if (cc->Inputs().HasTag(kBgraInTag) && cc->Outputs().HasTag(kRgbaOutTag)) {
return ConvertAndOutput(kBgraInTag, kRgbaOutTag, ImageFormat::SRGBA,
cv::COLOR_BGRA2RGBA, cc);
}
// RGBA -> BGRA
if (cc->Inputs().HasTag(kRgbaInTag) && cc->Outputs().HasTag(kBgraOutTag)) {
return ConvertAndOutput(kRgbaInTag, kBgraOutTag, ImageFormat::SBGRA,
cv::COLOR_RGBA2BGRA, cc);
}
return ::mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
<< "Unsupported image format conversion.";
mediapipe/calculators/image/image_cropping_calculator.cc
+1 -7
View File
@@ -514,13 +514,7 @@ RectSpec ImageCroppingCalculator::GetCropSpecs(const CalculatorContext* cc,
}
}
return {
.width = crop_width,
.height = crop_height,
.center_x = x_center,
.center_y = y_center,
.rotation = rotation,
};
return {crop_width, crop_height, x_center, y_center, rotation};
}
::mediapipe::Status ImageCroppingCalculator::GetBorderModeForOpenCV(
mediapipe/calculators/image/image_transformation_calculator.cc
+14 -10
View File
@@ -392,19 +392,26 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
}
cv::Mat scaled_mat;
int output_width = output_width_;
int output_height = output_height_;
if (scale_mode_ == mediapipe::ScaleMode_Mode_STRETCH) {
cv::resize(input_mat, scaled_mat, cv::Size(output_width_, output_height_));
int scale_flag =
input_mat.cols > output_width_ && input_mat.rows > output_height_
? cv::INTER_AREA
: cv::INTER_LINEAR;
cv::resize(input_mat, scaled_mat, cv::Size(output_width_, output_height_),
0, 0, scale_flag);
} else {
const float scale =
std::min(static_cast<float>(output_width_) / input_width,
static_cast<float>(output_height_) / input_height);
const int target_width = std::round(input_width * scale);
const int target_height = std::round(input_height * scale);
int scale_flag = scale < 1.0f ? cv::INTER_AREA : cv::INTER_LINEAR;
if (scale_mode_ == mediapipe::ScaleMode_Mode_FIT) {
cv::Mat intermediate_mat;
cv::resize(input_mat, intermediate_mat,
cv::Size(target_width, target_height));
cv::Size(target_width, target_height), 0, 0, scale_flag);
const int top = (output_height_ - target_height) / 2;
const int bottom = output_height_ - target_height - top;
const int left = (output_width_ - target_width) / 2;
@@ -413,16 +420,13 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
options_.constant_padding() ? cv::BORDER_CONSTANT
: cv::BORDER_REPLICATE);
} else {
cv::resize(input_mat, scaled_mat, cv::Size(target_width, target_height));
output_width_ = target_width;
output_height_ = target_height;
cv::resize(input_mat, scaled_mat, cv::Size(target_width, target_height),
0, 0, scale_flag);
output_width = target_width;
output_height = target_height;
}
}
int output_width;
int output_height;
ComputeOutputDimensions(input_width, input_height, &output_width,
&output_height);
if (cc->Outputs().HasTag("LETTERBOX_PADDING")) {
auto padding = absl::make_unique<std::array<float, 4>>();
ComputeOutputLetterboxPadding(input_width, input_height, output_width,
mediapipe/calculators/tensorflow/BUILD
+15 -15
View File
@@ -321,7 +321,7 @@ cc_library(
"@org_tensorflow//tensorflow/core:framework",
],
"//mediapipe:android": [
"@org_tensorflow//tensorflow/core:android_lib_lite",
"@org_tensorflow//tensorflow/core:portable_tensorflow_lib_lite",
],
}),
alwayslink = 1,
@@ -343,7 +343,7 @@ cc_library(
"@org_tensorflow//tensorflow/core:framework",
],
"//mediapipe:android": [
"@org_tensorflow//tensorflow/core:android_lib_lite",
"@org_tensorflow//tensorflow/core:portable_tensorflow_lib_lite",
],
}),
alwayslink = 1,
@@ -449,10 +449,10 @@ cc_library(
"@org_tensorflow//tensorflow/core:framework",
],
"//mediapipe:android": [
"@org_tensorflow//tensorflow/core:android_tensorflow_lib_lite_nortti_lite_protos",
"@org_tensorflow//tensorflow/core:portable_tensorflow_lib_lite",
],
"//mediapipe:ios": [
"@org_tensorflow//tensorflow/core:ios_tensorflow_lib",
"@org_tensorflow//tensorflow/core:portable_tensorflow_lib",
],
}),
alwayslink = 1,
@@ -470,10 +470,10 @@ cc_library(
"@org_tensorflow//tensorflow/core:core",
],
"//mediapipe:android": [
"@org_tensorflow//tensorflow/core:android_tensorflow_lib_lite_nortti_lite_protos",
"@org_tensorflow//tensorflow/core:portable_tensorflow_lib_lite",
],
"//mediapipe:ios": [
"@org_tensorflow//tensorflow/core:ios_tensorflow_lib",
"@org_tensorflow//tensorflow/core:portable_tensorflow_lib",
],
}),
)
@@ -496,11 +496,11 @@ cc_library(
"@org_tensorflow//tensorflow/core:core",
],
"//mediapipe:android": [
"@org_tensorflow//tensorflow/core:android_tensorflow_lib_lite_nortti_lite_protos",
"@org_tensorflow//tensorflow/core:portable_tensorflow_lib_lite",
"//mediapipe/android/file/base",
],
"//mediapipe:ios": [
"@org_tensorflow//tensorflow/core:ios_tensorflow_lib",
"@org_tensorflow//tensorflow/core:portable_tensorflow_lib",
"//mediapipe/android/file/base",
],
}),
@@ -525,11 +525,11 @@ cc_library(
"@org_tensorflow//tensorflow/core:core",
],
"//mediapipe:android": [
"@org_tensorflow//tensorflow/core:android_tensorflow_lib_lite_nortti_lite_protos",
"@org_tensorflow//tensorflow/core:portable_tensorflow_lib_lite",
"//mediapipe/android/file/base",
],
"//mediapipe:ios": [
"@org_tensorflow//tensorflow/core:ios_tensorflow_lib",
"@org_tensorflow//tensorflow/core:portable_tensorflow_lib",
"//mediapipe/android/file/base",
],
}),
@@ -637,7 +637,7 @@ cc_library(
"@org_tensorflow//tensorflow/core:framework",
],
"//mediapipe:android": [
"@org_tensorflow//tensorflow/core:android_lib_lite",
"@org_tensorflow//tensorflow/core:portable_tensorflow_lib_lite",
],
}),
alwayslink = 1,
@@ -673,7 +673,7 @@ cc_library(
"@org_tensorflow//tensorflow/core:framework",
],
"//mediapipe:android": [
"@org_tensorflow//tensorflow/core:android_lib_lite",
"@org_tensorflow//tensorflow/core:portable_tensorflow_lib_lite",
],
}),
alwayslink = 1,
@@ -1109,11 +1109,11 @@ cc_test(
"@org_tensorflow//tensorflow/core:direct_session",
],
"//mediapipe:android": [
"@org_tensorflow//tensorflow/core:android_tensorflow_lib_with_ops_lite_proto_no_rtti_lib",
"@org_tensorflow//tensorflow/core:android_tensorflow_test_lib",
"@org_tensorflow//tensorflow/core:portable_tensorflow_lib",
"@org_tensorflow//tensorflow/core:portable_tensorflow_test_lib",
],
"//mediapipe:ios": [
"@org_tensorflow//tensorflow/core:ios_tensorflow_test_lib",
"@org_tensorflow//tensorflow/core:portable_tensorflow_test_lib",
],
}),
)
mediapipe/calculators/tflite/BUILD
+4 -3
View File
@@ -198,6 +198,7 @@ cc_test(
cc_library(
name = "util",
hdrs = ["util.h"],
visibility = ["//visibility:public"],
alwayslink = 1,
)
@@ -525,16 +526,16 @@ cc_test(
":tflite_converter_calculator_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:calculator_runner",
"//mediapipe/framework/deps:file_path",
"//mediapipe/framework/formats:image_format_cc_proto",
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/formats:image_frame_opencv",
"//mediapipe/framework/formats:matrix",
"//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:integral_types",
"//mediapipe/framework/port:parse_text_proto",
"//mediapipe/framework/port:status",
"//mediapipe/framework/tool:validate_type",
"@com_google_absl//absl/memory",
"@org_tensorflow//tensorflow/lite:framework",
"@org_tensorflow//tensorflow/lite/kernels:builtin_ops",
],
)
mediapipe/calculators/tflite/ssd_anchors_calculator.cc
+7 -3
View File
@@ -26,8 +26,12 @@ namespace {
float CalculateScale(float min_scale, float max_scale, int stride_index,
int num_strides) {
return min_scale +
(max_scale - min_scale) * 1.0 * stride_index / (num_strides - 1.0f);
if (num_strides == 1) {
return (min_scale + max_scale) * 0.5f;
} else {
return min_scale +
(max_scale - min_scale) * 1.0 * stride_index / (num_strides - 1.0f);
}
}
} // namespace
@@ -114,7 +118,7 @@ REGISTER_CALCULATOR(SsdAnchorsCalculator);
}
int layer_id = 0;
while (layer_id < options.strides_size()) {
while (layer_id < options.num_layers()) {
std::vector<float> anchor_height;
std::vector<float> anchor_width;
std::vector<float> aspect_ratios;
mediapipe/calculators/tflite/tflite_converter_calculator.cc
+34 -7
View File
@@ -67,10 +67,12 @@ constexpr char kImageFrameTag[] = "IMAGE";
constexpr char kGpuBufferTag[] = "IMAGE_GPU";
constexpr char kTensorsTag[] = "TENSORS";
constexpr char kTensorsGpuTag[] = "TENSORS_GPU";
constexpr char kMatrixTag[] = "MATRIX";
} // namespace
namespace mediapipe {
namespace {
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
using ::tflite::gpu::gl::CreateReadWriteShaderStorageBuffer;
using ::tflite::gpu::gl::GlProgram;
@@ -89,6 +91,8 @@ struct GPUData {
};
#endif
} // namespace
// Calculator for normalizing and converting an ImageFrame or Matrix
// into a TfLiteTensor (float 32) or a GpuBuffer to a tflite::gpu::GlBuffer
// or MTLBuffer.
@@ -164,6 +168,9 @@ class TfLiteConverterCalculator : public CalculatorBase {
bool initialized_ = false;
bool use_gpu_ = false;
bool zero_center_ = true; // normalize range to [-1,1] | otherwise [0,1]
bool use_custom_normalization_ = false;
float custom_div_ = -1.0f;
float custom_sub_ = -1.0f;
bool flip_vertically_ = false;
bool row_major_matrix_ = false;
bool use_quantized_tensors_ = false;
@@ -175,7 +182,8 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
CalculatorContract* cc) {
// Confirm only one of the input streams is present.
RET_CHECK(cc->Inputs().HasTag(kImageFrameTag) ^
cc->Inputs().HasTag(kGpuBufferTag) ^ cc->Inputs().HasTag("MATRIX"));
cc->Inputs().HasTag(kGpuBufferTag) ^
cc->Inputs().HasTag(kMatrixTag));
// Confirm only one of the output streams is present.
RET_CHECK(cc->Outputs().HasTag(kTensorsTag) ^
@@ -186,8 +194,8 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
if (cc->Inputs().HasTag(kImageFrameTag)) {
cc->Inputs().Tag(kImageFrameTag).Set<ImageFrame>();
}
if (cc->Inputs().HasTag("MATRIX")) {
cc->Inputs().Tag("MATRIX").Set<Matrix>();
if (cc->Inputs().HasTag(kMatrixTag)) {
cc->Inputs().Tag(kMatrixTag).Set<Matrix>();
}
#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__)
if (cc->Inputs().HasTag(kGpuBufferTag)) {
@@ -257,6 +265,9 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
::mediapipe::Status TfLiteConverterCalculator::Process(CalculatorContext* cc) {
if (use_gpu_) {
if (cc->Inputs().Tag(kGpuBufferTag).IsEmpty()) {
return ::mediapipe::OkStatus();
}
if (!initialized_) {
MP_RETURN_IF_ERROR(InitGpu(cc));
initialized_ = true;
@@ -283,6 +294,9 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
::mediapipe::Status TfLiteConverterCalculator::ProcessCPU(
CalculatorContext* cc) {
if (cc->Inputs().HasTag(kImageFrameTag)) {
if (cc->Inputs().Tag(kImageFrameTag).IsEmpty()) {
return ::mediapipe::OkStatus();
}
// CPU ImageFrame to TfLiteTensor conversion.
const auto& image_frame =
@@ -361,10 +375,12 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
cc->Outputs()
.Tag(kTensorsTag)
.Add(output_tensors.release(), cc->InputTimestamp());
} else if (cc->Inputs().HasTag("MATRIX")) {
} else if (cc->Inputs().HasTag(kMatrixTag)) {
if (cc->Inputs().Tag(kMatrixTag).IsEmpty()) {
return ::mediapipe::OkStatus();
}
// CPU Matrix to TfLiteTensor conversion.
const auto& matrix = cc->Inputs().Tag("MATRIX").Get<Matrix>();
const auto& matrix = cc->Inputs().Tag(kMatrixTag).Get<Matrix>();
const int height = matrix.rows();
const int width = matrix.cols();
const int channels = 1;
@@ -614,6 +630,11 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
// Get data normalization mode.
zero_center_ = options.zero_center();
// Custom div and sub values.
use_custom_normalization_ = options.use_custom_normalization();
custom_div_ = options.custom_div();
custom_sub_ = options.custom_sub();
// Get y-flip mode.
flip_vertically_ = options.flip_vertically();
@@ -649,7 +670,13 @@ template <class T>
const int channels_ignored = channels - channels_preserved;
float div, sub;
if (zero_center) {
if (use_custom_normalization_) {
RET_CHECK_GT(custom_div_, 0.0f);
RET_CHECK_GE(custom_sub_, 0.0f);
div = custom_div_;
sub = custom_sub_;
} else if (zero_center) {
// [-1,1]
div = 127.5f;
sub = 1.0f;
mediapipe/calculators/tflite/tflite_converter_calculator.proto
+10
View File
@@ -28,6 +28,16 @@ message TfLiteConverterCalculatorOptions {
// Ignored if using quantization.
optional bool zero_center = 1 [default = true];
// Custom settings to override the internal scaling factors `div` and `sub`.
// Both values must be set to non-negative values. Will only take effect on
// CPU AND when |use_custom_normalization| is set to true. When these custom
// values take effect, the |zero_center| setting above will be overriden, and
// the normalized_value will be calculated as:
// normalized_value = input / custom_div - custom_sub.
optional bool use_custom_normalization = 6 [default = false];
optional float custom_div = 7 [default = -1.0];
optional float custom_sub = 8 [default = -1.0];
// Whether the input image should be flipped vertically (along the
// y-direction). This is useful, for example, when the input image is defined
// with a coordinate system where the origin is at the bottom-left corner
mediapipe/calculators/tflite/tflite_converter_calculator_test.cc
+54 -1
View File
@@ -19,6 +19,9 @@
#include "mediapipe/calculators/tflite/tflite_converter_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_runner.h"
#include "mediapipe/framework/formats/image_format.pb.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/formats/image_frame_opencv.h"
#include "mediapipe/framework/formats/matrix.h"
#include "mediapipe/framework/port/gtest.h"
#include "mediapipe/framework/port/integral_types.h"
@@ -28,7 +31,6 @@
#include "tensorflow/lite/interpreter.h"
namespace mediapipe {
namespace {
constexpr char kTransposeOptionsString[] =
@@ -196,4 +198,55 @@ TEST_F(TfLiteConverterCalculatorTest, RandomMatrixRowMajor) {
}
}
TEST_F(TfLiteConverterCalculatorTest, CustomDivAndSub) {
CalculatorGraph graph;
// Run the calculator and verify that one output is generated.
CalculatorGraphConfig graph_config =
::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
input_stream: "input_image"
node {
calculator: "TfLiteConverterCalculator"
input_stream: "IMAGE:input_image"
output_stream: "TENSORS:tensor"
options {
[mediapipe.TfLiteConverterCalculatorOptions.ext] {
row_major_matrix: true
use_custom_normalization: true
custom_div: 2.0
custom_sub: 33.0
}
}
}
)");
std::vector<Packet> output_packets;
tool::AddVectorSink("tensor", &graph_config, &output_packets);
// Run the graph.
MP_ASSERT_OK(graph.Initialize(graph_config));
MP_ASSERT_OK(graph.StartRun({}));
auto input_image = absl::make_unique<ImageFrame>(ImageFormat::GRAY8, 1, 1);
cv::Mat mat = ::mediapipe::formats::MatView(input_image.get());
mat.at<uint8>(0, 0) = 200;
MP_ASSERT_OK(graph.AddPacketToInputStream(
"input_image", Adopt(input_image.release()).At(Timestamp(0))));
// Wait until the calculator done processing.
MP_ASSERT_OK(graph.WaitUntilIdle());
EXPECT_EQ(1, output_packets.size());
// Get and process results.
const std::vector<TfLiteTensor>& tensor_vec =
output_packets[0].Get<std::vector<TfLiteTensor>>();
EXPECT_EQ(1, tensor_vec.size());
const TfLiteTensor* tensor = &tensor_vec[0];
EXPECT_EQ(kTfLiteFloat32, tensor->type);
EXPECT_FLOAT_EQ(67.0f, *tensor->data.f);
// Fully close graph at end, otherwise calculator+tensors are destroyed
// after calling WaitUntilDone().
MP_ASSERT_OK(graph.CloseInputStream("input_image"));
MP_ASSERT_OK(graph.WaitUntilDone());
}
} // namespace mediapipe
mediapipe/calculators/tflite/tflite_inference_calculator.cc
+23 -4
View File
@@ -57,7 +57,10 @@
#include "tensorflow/lite/delegates/gpu/metal_delegate.h"
#include "tensorflow/lite/delegates/gpu/metal_delegate_internal.h"
#endif // iOS
#if !defined(MEDIAPIPE_EDGE_TPU)
#include "tensorflow/lite/delegates/xnnpack/xnnpack_delegate.h"
#endif // !EDGETPU
#if defined(MEDIAPIPE_ANDROID)
#include "tensorflow/lite/delegates/nnapi/nnapi_delegate.h"
#endif // ANDROID
@@ -116,11 +119,13 @@ using ::tflite::gpu::gl::GlBuffer;
#endif
#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__)
namespace {
struct GPUData {
int elements = 1;
GpuTensor buffer;
::tflite::gpu::BHWC shape;
};
} // namespace
#endif
// Returns number of threads to configure XNNPACK delegate with.
@@ -405,8 +410,11 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
// 1. Receive pre-processed tensor inputs.
if (use_advanced_gpu_api_) {
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
if (cc->Inputs().Tag(kTensorsGpuTag).IsEmpty()) {
return ::mediapipe::OkStatus();
}
const auto& input_tensors =
cc->Inputs().Tag("TENSORS_GPU").Get<std::vector<GpuTensor>>();
cc->Inputs().Tag(kTensorsGpuTag).Get<std::vector<GpuTensor>>();
RET_CHECK(!input_tensors.empty());
MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext(
[this, &input_tensors]() -> ::mediapipe::Status {
@@ -424,6 +432,9 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
} else if (gpu_input_) {
// Read GPU input into SSBO.
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
if (cc->Inputs().Tag(kTensorsGpuTag).IsEmpty()) {
return ::mediapipe::OkStatus();
}
const auto& input_tensors =
cc->Inputs().Tag(kTensorsGpuTag).Get<std::vector<GpuTensor>>();
RET_CHECK_GT(input_tensors.size(), 0);
@@ -439,6 +450,9 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
return ::mediapipe::OkStatus();
}));
#elif defined(MEDIAPIPE_IOS)
if (cc->Inputs().Tag(kTensorsGpuTag).IsEmpty()) {
return ::mediapipe::OkStatus();
}
const auto& input_tensors =
cc->Inputs().Tag(kTensorsGpuTag).Get<std::vector<GpuTensor>>();
RET_CHECK_GT(input_tensors.size(), 0);
@@ -465,6 +479,9 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
RET_CHECK_FAIL() << "GPU processing not enabled.";
#endif
} else {
if (cc->Inputs().Tag(kTensorsTag).IsEmpty()) {
return ::mediapipe::OkStatus();
}
// Read CPU input into tensors.
const auto& input_tensors =
cc->Inputs().Tag(kTensorsTag).Get<std::vector<TfLiteTensor>>();
@@ -511,10 +528,10 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
auto output_tensors = absl::make_unique<std::vector<GpuTensor>>();
output_tensors->resize(gpu_data_out_.size());
for (int i = 0; i < gpu_data_out_.size(); ++i) {
output_tensors->at(i) = gpu_data_out_[0]->buffer.MakeRef();
output_tensors->at(i) = gpu_data_out_[i]->buffer.MakeRef();
}
cc->Outputs()
.Tag("TENSORS_GPU")
.Tag(kTensorsGpuTag)
.Add(output_tensors.release(), cc->InputTimestamp());
#endif
} else if (gpu_output_) {
@@ -637,7 +654,7 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
options.usage = tflite::gpu::InferenceUsage::SUSTAINED_SPEED;
tflite_gpu_runner_ =
std::make_unique<tflite::gpu::TFLiteGPURunner>(options);
return tflite_gpu_runner_->InitializeWithModel(model);
return tflite_gpu_runner_->InitializeWithModel(model, op_resolver);
}
#endif
@@ -730,6 +747,7 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
calculator_opts.delegate().has_xnnpack();
#endif // __EMSCRIPTEN__
#if !defined(MEDIAPIPE_EDGE_TPU)
if (xnnpack_requested) {
TfLiteXNNPackDelegateOptions xnnpack_opts{};
xnnpack_opts.num_threads = GetXnnpackNumThreads(calculator_opts);
@@ -738,6 +756,7 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
RET_CHECK_EQ(interpreter_->ModifyGraphWithDelegate(delegate_.get()),
kTfLiteOk);
}
#endif // !EDGETPU
// Return, no need for GPU delegate below.
return ::mediapipe::OkStatus();
mediapipe/calculators/tflite/tflite_tensors_to_segmentation_calculator.cc
+4 -1
View File
@@ -77,7 +77,10 @@ using ::tflite::gpu::gl::GlShader;
// Performs optional upscale to REFERENCE_IMAGE dimensions if provided,
// otherwise the mask is the same size as input tensor.
//
// Produces result as an RGBA image, with the mask in both R & A channels.
// Produces result as an RGBA image, with the mask in both R & A channels. The
// value of each pixel is the probability of the specified class after softmax,
// scaled to 255 on CPU. The class can be specified through the
// |output_layer_index| option.
//
// Inputs:
// One of the following TENSORS tags:
mediapipe/calculators/util/BUILD
+35
View File
@@ -276,6 +276,41 @@ cc_test(
],
)
cc_library(
name = "clock_timestamp_calculator",
srcs = ["clock_timestamp_calculator.cc"],
visibility = [
"//visibility:public",
],
deps = [
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:timestamp",
"//mediapipe/framework/deps:clock",
"//mediapipe/framework/port:logging",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"@com_google_absl//absl/time",
],
alwayslink = 1,
)
cc_library(
name = "clock_latency_calculator",
srcs = ["clock_latency_calculator.cc"],
visibility = [
"//visibility:public",
],
deps = [
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:timestamp",
"//mediapipe/framework/port:logging",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"@com_google_absl//absl/time",
],
alwayslink = 1,
)
cc_library(
name = "annotation_overlay_calculator",
srcs = ["annotation_overlay_calculator.cc"],
mediapipe/calculators/util/clock_latency_calculator.cc
+116
View File
@@ -0,0 +1,116 @@
// Copyright 2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "absl/time/time.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/port/logging.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status.h"
namespace mediapipe {
namespace {
// Tag name for reference signal.
constexpr char kReferenceTag[] = "REFERENCE";
} // namespace
// A calculator that diffs multiple input absl::Time streams against a
// reference Time stream, and outputs the resulting absl::Duration's. Useful
// in combination with ClockTimestampCalculator to be able to determine the
// latency between two different points in a graph.
//
// Inputs: At least one non-reference Time stream is required.
// 0- Time stream 0
// 1- Time stream 1
// ...
// N- Time stream N
// REFERENCE_SIGNAL (required): The Time stream by which all others are
// compared. Should be the stream from which our other streams were
// computed, in order to provide meaningful latency results.
//
// Outputs:
// 0- Duration from REFERENCE_SIGNAL to input stream 0
// 1- Duration from REFERENCE_SIGNAL to input stream 1
// ...
// N- Duration from REFERENCE_SIGNAL to input stream N
//
// Example config:
// node {
// calculator: "ClockLatencyCalculator"
// input_stream: "packet_clocktime_stream_0"
// input_stream: "packet_clocktime_stream_1"
// input_stream: "packet_clocktime_stream_2"
// input_stream: "REFERENCE_SIGNAL: packet_clocktime_stream_reference"
// output_stream: "packet_latency_stream_0"
// output_stream: "packet_latency_stream_1"
// output_stream: "packet_latency_stream_2"
// }
//
class ClockLatencyCalculator : public CalculatorBase {
public:
ClockLatencyCalculator() {}
static ::mediapipe::Status GetContract(CalculatorContract* cc);
::mediapipe::Status Open(CalculatorContext* cc) override;
::mediapipe::Status Process(CalculatorContext* cc) override;
private:
int64 num_packet_streams_ = -1;
};
REGISTER_CALCULATOR(ClockLatencyCalculator);
::mediapipe::Status ClockLatencyCalculator::GetContract(
CalculatorContract* cc) {
RET_CHECK_GT(cc->Inputs().NumEntries(), 1);
int64 num_packet_streams = cc->Inputs().NumEntries() - 1;
RET_CHECK_EQ(cc->Outputs().NumEntries(), num_packet_streams);
for (int64 i = 0; i < num_packet_streams; ++i) {
cc->Inputs().Index(i).Set<absl::Time>();
cc->Outputs().Index(i).Set<absl::Duration>();
}
cc->Inputs().Tag(kReferenceTag).Set<absl::Time>();
return ::mediapipe::OkStatus();
}
::mediapipe::Status ClockLatencyCalculator::Open(CalculatorContext* cc) {
// Direct passthrough, as far as timestamp and bounds are concerned.
cc->SetOffset(TimestampDiff(0));
num_packet_streams_ = cc->Inputs().NumEntries() - 1;
return ::mediapipe::OkStatus();
}
::mediapipe::Status ClockLatencyCalculator::Process(CalculatorContext* cc) {
// Get reference time.
RET_CHECK(!cc->Inputs().Tag(kReferenceTag).IsEmpty());
const absl::Time& reference_time =
cc->Inputs().Tag(kReferenceTag).Get<absl::Time>();
// Push Duration packets for every input stream we have.
for (int64 i = 0; i < num_packet_streams_; ++i) {
if (!cc->Inputs().Index(i).IsEmpty()) {
const absl::Time& input_stream_time =
cc->Inputs().Index(i).Get<absl::Time>();
cc->Outputs().Index(i).AddPacket(
MakePacket<absl::Duration>(input_stream_time - reference_time)
.At(cc->InputTimestamp()));
}
}
return ::mediapipe::OkStatus();
}
} // namespace mediapipe
mediapipe/calculators/util/clock_timestamp_calculator.cc
+108
View File
@@ -0,0 +1,108 @@
// Copyright 2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "absl/time/time.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/deps/clock.h"
#include "mediapipe/framework/deps/monotonic_clock.h"
#include "mediapipe/framework/port/logging.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status.h"
namespace mediapipe {
namespace {
// Tag name for clock side packet.
constexpr char kClockTag[] = "CLOCK";
} // namespace
// A calculator that outputs the current clock time at which it receives input
// packets. Use a separate instance of this calculator for each input stream
// you wish to output a clock time for.
//
// InputSidePacket (Optional):
// CLOCK: A clock to use for querying the current time.
//
// Inputs:
// A single packet stream we wish to get the current clocktime for
// Outputs:
// A single stream of absl::Time packets, representing the clock time at which
// we received the input stream's packets.
// Example config:
// node {
// calculator: "ClockTimestampCalculator"
// input_side_packet: "CLOCK:monotonic_clock"
// input_stream: "packet_stream"
// output_stream: "packet_clocktime_stream"
// }
//
class ClockTimestampCalculator : public CalculatorBase {
public:
ClockTimestampCalculator() {}
static ::mediapipe::Status GetContract(CalculatorContract* cc);
::mediapipe::Status Open(CalculatorContext* cc) override;
::mediapipe::Status Process(CalculatorContext* cc) override;
private:
// Clock object.
std::shared_ptr<::mediapipe::Clock> clock_;
};
REGISTER_CALCULATOR(ClockTimestampCalculator);
::mediapipe::Status ClockTimestampCalculator::GetContract(
CalculatorContract* cc) {
RET_CHECK_EQ(cc->Inputs().NumEntries(), 1);
RET_CHECK_EQ(cc->Outputs().NumEntries(), 1);
cc->Inputs().Index(0).SetAny();
cc->Outputs().Index(0).Set<absl::Time>();
// Optional Clock input side packet.
if (cc->InputSidePackets().HasTag(kClockTag)) {
cc->InputSidePackets()
.Tag(kClockTag)
.Set<std::shared_ptr<::mediapipe::Clock>>();
}
return ::mediapipe::OkStatus();
}
::mediapipe::Status ClockTimestampCalculator::Open(CalculatorContext* cc) {
// Direct passthrough, as far as timestamp and bounds are concerned.
cc->SetOffset(TimestampDiff(0));
// Initialize the clock.
if (cc->InputSidePackets().HasTag(kClockTag)) {
clock_ = cc->InputSidePackets()
.Tag("CLOCK")
.Get<std::shared_ptr<::mediapipe::Clock>>();
} else {
clock_.reset(
::mediapipe::MonotonicClock::CreateSynchronizedMonotonicClock());
}
return ::mediapipe::OkStatus();
}
::mediapipe::Status ClockTimestampCalculator::Process(CalculatorContext* cc) {
// Push the Time packet to output.
auto timestamp_packet = MakePacket<absl::Time>(clock_->TimeNow());
cc->Outputs().Index(0).AddPacket(timestamp_packet.At(cc->InputTimestamp()));
return ::mediapipe::OkStatus();
}
} // namespace mediapipe
mediapipe/calculators/util/detections_to_render_data_calculator.cc
+14 -2
View File
@@ -27,6 +27,7 @@ namespace mediapipe {
namespace {
constexpr char kDetectionTag[] = "DETECTION";
constexpr char kDetectionsTag[] = "DETECTIONS";
constexpr char kDetectionListTag[] = "DETECTION_LIST";
constexpr char kRenderDataTag[] = "RENDER_DATA";
@@ -62,6 +63,7 @@ constexpr float kNumScoreDecimalDigitsMultipler = 100;
// Example config:
// node {
// calculator: "DetectionsToRenderDataCalculator"
// input_stream: "DETECTION:detection"
// input_stream: "DETECTIONS:detections"
// input_stream: "DETECTION_LIST:detection_list"
// output_stream: "RENDER_DATA:render_data"
@@ -123,9 +125,13 @@ REGISTER_CALCULATOR(DetectionsToRenderDataCalculator);
::mediapipe::Status DetectionsToRenderDataCalculator::GetContract(
CalculatorContract* cc) {
RET_CHECK(cc->Inputs().HasTag(kDetectionListTag) ||
cc->Inputs().HasTag(kDetectionsTag))
cc->Inputs().HasTag(kDetectionsTag) ||
cc->Inputs().HasTag(kDetectionTag))
<< "None of the input streams are provided.";
if (cc->Inputs().HasTag(kDetectionTag)) {
cc->Inputs().Tag(kDetectionTag).Set<Detection>();
}
if (cc->Inputs().HasTag(kDetectionListTag)) {
cc->Inputs().Tag(kDetectionListTag).Set<DetectionList>();
}
@@ -155,8 +161,10 @@ REGISTER_CALCULATOR(DetectionsToRenderDataCalculator);
const bool has_detection_from_vector =
cc->Inputs().HasTag(kDetectionsTag) &&
!cc->Inputs().Tag(kDetectionsTag).Get<std::vector<Detection>>().empty();
const bool has_single_detection = cc->Inputs().HasTag(kDetectionTag) &&
!cc->Inputs().Tag(kDetectionTag).IsEmpty();
if (!options.produce_empty_packet() && !has_detection_from_list &&
!has_detection_from_vector) {
!has_detection_from_vector && !has_single_detection) {
return ::mediapipe::OkStatus();
}
@@ -176,6 +184,10 @@ REGISTER_CALCULATOR(DetectionsToRenderDataCalculator);
AddDetectionToRenderData(detection, options, render_data.get());
}
}
if (has_single_detection) {
AddDetectionToRenderData(cc->Inputs().Tag(kDetectionTag).Get<Detection>(),
options, render_data.get());
}
cc->Outputs()
.Tag(kRenderDataTag)
.Add(render_data.release(), cc->InputTimestamp());
mediapipe/calculators/util/landmarks_to_detection_calculator.cc
+1 -1
View File
@@ -76,7 +76,7 @@ Detection ConvertLandmarksToDetection(const NormalizedLandmarkList& landmarks) {
// node {
// calculator: "LandmarksToDetectionCalculator"
// input_stream: "NORM_LANDMARKS:landmarks"
// output_stream: "DETECTIONS:detections"
// output_stream: "DETECTION:detections"
// }
class LandmarksToDetectionCalculator : public CalculatorBase {
public:
mediapipe/calculators/util/non_max_suppression_calculator.cc
+9 -2
View File
@@ -303,12 +303,12 @@ class NonMaxSuppressionCalculator : public CalculatorBase {
IndexedScores candidates;
output_detections->clear();
while (!remained_indexed_scores.empty()) {
const int original_indexed_scores_size = remained_indexed_scores.size();
const auto& detection = detections[remained_indexed_scores[0].first];
if (options_.min_score_threshold() > 0 &&
detection.score(0) < options_.min_score_threshold()) {
break;
}
remained.clear();
candidates.clear();
const Location location(detection.location_data());
@@ -365,8 +365,15 @@ class NonMaxSuppressionCalculator : public CalculatorBase {
keypoint->set_y(keypoints[i * 2 + 1] / total_score);
}
}
remained_indexed_scores = std::move(remained);
output_detections->push_back(weighted_detection);
// Breaks the loop if the size of indexed scores doesn't change after an
// iteration.
if (original_indexed_scores_size == remained.size()) {
break;
} else {
remained_indexed_scores = std::move(remained);
}
}
}