talksik/mediapipe
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Project import generated by Copybara.

Browse Source 
GitOrigin-RevId: 73d686c40057684f8bfaca285368bf1813f9fc26
This commit is contained in:
MediaPipe Team
2022-03-21 12:12:39 -07:00
committed by jqtang
parent e6c19885c6
commit cc6a2f7af6
266 changed files with 3658 additions and 1681 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mediapipe/calculators/tensor/BUILD
+8 -7
View File
@@ -84,14 +84,15 @@ cc_library(
tags = ["nomac"], # config problem with cpuinfo via TF
deps = [
"inference_calculator_interface",
"//mediapipe/framework/deps:file_path",
"//mediapipe/gpu:gl_calculator_helper",
"//mediapipe/gpu:gpu_buffer",
"//mediapipe/util/tflite:config",
"//mediapipe/util/tflite:tflite_gpu_runner",
"@com_google_absl//absl/memory",
"@com_google_absl//absl/status",
"@org_tensorflow//tensorflow/lite/delegates/gpu:gl_delegate",
"@org_tensorflow//tensorflow/lite/delegates/gpu/common:shape",
"@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_buffer",
"@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_program",
"@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_shader",
],
alwayslink = 1,
)
@@ -154,7 +155,7 @@ cc_library(
cc_library(
name = "inference_calculator_gl_if_compute_shader_available",
deps = select({
deps = selects.with_or({
":compute_shader_unavailable": [],
"//conditions:default": [":inference_calculator_gl"],
}),
@@ -303,7 +304,7 @@ cc_library(
"//mediapipe/framework/formats:tensor",
"//mediapipe/framework/formats/object_detection:anchor_cc_proto",
"//mediapipe/framework/port:ret_check",
] + select({
] + selects.with_or({
":compute_shader_unavailable": [],
"//conditions:default": [":tensors_to_detections_calculator_gpu_deps"],
}),
@@ -560,7 +561,7 @@ cc_library(
cc_library(
name = "image_to_tensor_calculator_gpu_deps",
deps = select({
deps = selects.with_or({
"//mediapipe:android": [
":image_to_tensor_converter_gl_buffer",
"//mediapipe/gpu:gl_calculator_helper",
@@ -684,7 +685,7 @@ cc_library(
name = "image_to_tensor_converter_gl_buffer",
srcs = ["image_to_tensor_converter_gl_buffer.cc"],
hdrs = ["image_to_tensor_converter_gl_buffer.h"],
deps = ["//mediapipe/framework:port"] + select({
deps = ["//mediapipe/framework:port"] + selects.with_or({
"//mediapipe:apple": [],
"//conditions:default": [
":image_to_tensor_converter",
mediapipe/calculators/tensor/image_to_tensor_calculator.cc
+51 -15
View File
@@ -49,7 +49,6 @@
#include "mediapipe/calculators/tensor/image_to_tensor_converter_gl_texture.h"
#include "mediapipe/gpu/gl_calculator_helper.h"
#endif // MEDIAPIPE_METAL_ENABLED
#endif // !MEDIAPIPE_DISABLE_GPU
namespace mediapipe {
@@ -142,11 +141,24 @@ class ImageToTensorCalculator : public Node {
const auto& options =
cc->Options<mediapipe::ImageToTensorCalculatorOptions>();
RET_CHECK(options.has_output_tensor_float_range())
RET_CHECK(options.has_output_tensor_float_range() ||
options.has_output_tensor_int_range())
<< "Output tensor range is required.";
RET_CHECK_LT(options.output_tensor_float_range().min(),
options.output_tensor_float_range().max())
<< "Valid output tensor range is required.";
if (options.has_output_tensor_float_range()) {
RET_CHECK_LT(options.output_tensor_float_range().min(),
options.output_tensor_float_range().max())
<< "Valid output float tensor range is required.";
}
if (options.has_output_tensor_int_range()) {
RET_CHECK_LT(options.output_tensor_int_range().min(),
options.output_tensor_int_range().max())
<< "Valid output int tensor range is required.";
RET_CHECK_GE(options.output_tensor_int_range().min(), 0)
<< "The minimum of the output int tensor range must be non-negative.";
RET_CHECK_LE(options.output_tensor_int_range().max(), 255)
<< "The maximum of the output int tensor range must be less than or "
"equal to 255.";
}
RET_CHECK_GT(options.output_tensor_width(), 0)
<< "Valid output tensor width is required.";
RET_CHECK_GT(options.output_tensor_height(), 0)
@@ -175,9 +187,15 @@ class ImageToTensorCalculator : public Node {
options_ = cc->Options<mediapipe::ImageToTensorCalculatorOptions>();
output_width_ = options_.output_tensor_width();
output_height_ = options_.output_tensor_height();
range_min_ = options_.output_tensor_float_range().min();
range_max_ = options_.output_tensor_float_range().max();
is_int_output_ = options_.has_output_tensor_int_range();
range_min_ =
is_int_output_
? static_cast<float>(options_.output_tensor_int_range().min())
: options_.output_tensor_float_range().min();
range_max_ =
is_int_output_
? static_cast<float>(options_.output_tensor_int_range().max())
: options_.output_tensor_float_range().max();
return absl::OkStatus();
}
@@ -225,7 +243,7 @@ class ImageToTensorCalculator : public Node {
}
// Lazy initialization of the GPU or CPU converter.
MP_RETURN_IF_ERROR(InitConverterIfNecessary(cc, image->UsesGpu()));
MP_RETURN_IF_ERROR(InitConverterIfNecessary(cc, *image.get()));
ASSIGN_OR_RETURN(Tensor tensor,
(image->UsesGpu() ? gpu_converter_ : cpu_converter_)
@@ -283,9 +301,15 @@ class ImageToTensorCalculator : public Node {
}
}
absl::Status InitConverterIfNecessary(CalculatorContext* cc, bool use_gpu) {
absl::Status InitConverterIfNecessary(CalculatorContext* cc,
const Image& image) {
// Lazy initialization of the GPU or CPU converter.
if (use_gpu) {
if (image.UsesGpu()) {
if (is_int_output_) {
return absl::UnimplementedError(
"ImageToTensorConverter for the input GPU image currently doesn't "
"support quantization.");
}
if (!gpu_converter_) {
#if !MEDIAPIPE_DISABLE_GPU
#if MEDIAPIPE_METAL_ENABLED
@@ -296,9 +320,17 @@ class ImageToTensorCalculator : public Node {
CreateImageToGlBufferTensorConverter(
cc, DoesGpuInputStartAtBottom(), GetBorderMode()));
#else
ASSIGN_OR_RETURN(gpu_converter_,
CreateImageToGlTextureTensorConverter(
cc, DoesGpuInputStartAtBottom(), GetBorderMode()));
// Check whether the underlying storage object is a GL texture.
if (image.GetGpuBuffer()
.internal_storage<mediapipe::GlTextureBuffer>()) {
ASSIGN_OR_RETURN(
gpu_converter_,
CreateImageToGlTextureTensorConverter(
cc, DoesGpuInputStartAtBottom(), GetBorderMode()));
} else {
return absl::UnimplementedError(
"ImageToTensorConverter for the input GPU image is unavailable.");
}
#endif // MEDIAPIPE_METAL_ENABLED
#endif // !MEDIAPIPE_DISABLE_GPU
}
@@ -306,7 +338,10 @@ class ImageToTensorCalculator : public Node {
if (!cpu_converter_) {
#if !MEDIAPIPE_DISABLE_OPENCV
ASSIGN_OR_RETURN(cpu_converter_,
CreateOpenCvConverter(cc, GetBorderMode()));
CreateOpenCvConverter(
cc, GetBorderMode(),
is_int_output_ ? Tensor::ElementType::kUInt8
: Tensor::ElementType::kFloat32));
#else
LOG(FATAL) << "Cannot create image to tensor opencv converter since "
"MEDIAPIPE_DISABLE_OPENCV is defined.";
@@ -321,6 +356,7 @@ class ImageToTensorCalculator : public Node {
mediapipe::ImageToTensorCalculatorOptions options_;
int output_width_ = 0;
int output_height_ = 0;
bool is_int_output_ = false;
float range_min_ = 0.0f;
float range_max_ = 1.0f;
};
mediapipe/calculators/tensor/image_to_tensor_calculator.proto
+9
View File
@@ -31,6 +31,14 @@ message ImageToTensorCalculatorOptions {
optional float max = 2;
}
// Range of int values [min, max].
// min, must be strictly less than max.
// Please note that IntRange is supported for CPU tensors only.
message IntRange {
optional int64 min = 1;
optional int64 max = 2;
}
// Pixel extrapolation methods. See @border_mode.
enum BorderMode {
BORDER_UNSPECIFIED = 0;
@@ -49,6 +57,7 @@ message ImageToTensorCalculatorOptions {
// Output tensor element range/type image pixels are converted to.
oneof range {
FloatRange output_tensor_float_range = 4;
IntRange output_tensor_int_range = 7;
}
// For CONVENTIONAL mode for OpenGL, input image starts at bottom and needs
mediapipe/calculators/tensor/image_to_tensor_calculator_test.cc
+77 -46
View File
@@ -61,7 +61,8 @@ void RunTestWithInputImagePacket(const Packet& input_image_packet,
float range_max, int tensor_width,
int tensor_height, bool keep_aspect,
absl::optional<BorderMode> border_mode,
const mediapipe::NormalizedRect& roi) {
const mediapipe::NormalizedRect& roi,
bool output_int_tensor) {
std::string border_mode_str;
if (border_mode) {
switch (*border_mode) {
@@ -73,6 +74,21 @@ void RunTestWithInputImagePacket(const Packet& input_image_packet,
break;
}
}
std::string output_tensor_range;
if (output_int_tensor) {
output_tensor_range = absl::Substitute(R"(output_tensor_int_range {
min: $0
max: $1
})",
static_cast<int>(range_min),
static_cast<int>(range_max));
} else {
output_tensor_range = absl::Substitute(R"(output_tensor_float_range {
min: $0
max: $1
})",
range_min, range_max);
}
auto graph_config = mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
absl::Substitute(R"(
input_stream: "input_image"
@@ -86,22 +102,18 @@ void RunTestWithInputImagePacket(const Packet& input_image_packet,
[mediapipe.ImageToTensorCalculatorOptions.ext] {
output_tensor_width: $0
output_tensor_height: $1
keep_aspect_ratio: $4
output_tensor_float_range {
min: $2
max: $3
}
$5 # border mode
keep_aspect_ratio: $2
$3 # output range
$4 # border mode
}
}
}
)",
/*$0=*/tensor_width,
/*$1=*/tensor_height,
/*$2=*/range_min,
/*$3=*/range_max,
/*$4=*/keep_aspect ? "true" : "false",
/*$5=*/border_mode_str));
/*$2=*/keep_aspect ? "true" : "false",
/*$3=*/output_tensor_range,
/*$4=*/border_mode_str));
std::vector<Packet> output_packets;
tool::AddVectorSink("tensor", &graph_config, &output_packets);
@@ -126,11 +138,18 @@ void RunTestWithInputImagePacket(const Packet& input_image_packet,
ASSERT_THAT(tensor_vec, testing::SizeIs(1));
const Tensor& tensor = tensor_vec[0];
EXPECT_EQ(tensor.element_type(), Tensor::ElementType::kFloat32);
auto view = tensor.GetCpuReadView();
cv::Mat tensor_mat(tensor_height, tensor_width, CV_32FC3,
const_cast<float*>(view.buffer<float>()));
cv::Mat tensor_mat;
if (output_int_tensor) {
EXPECT_EQ(tensor.element_type(), Tensor::ElementType::kUInt8);
tensor_mat = cv::Mat(tensor_height, tensor_width, CV_8UC3,
const_cast<uint8*>(view.buffer<uint8>()));
} else {
EXPECT_EQ(tensor.element_type(), Tensor::ElementType::kFloat32);
tensor_mat = cv::Mat(tensor_height, tensor_width, CV_32FC3,
const_cast<float*>(view.buffer<float>()));
}
cv::Mat result_rgb;
auto transformation =
GetValueRangeTransformation(range_min, range_max, 0.0f, 255.0f).value();
@@ -170,16 +189,26 @@ enum class InputType { kImageFrame, kImage };
const std::vector<InputType> kInputTypesToTest = {InputType::kImageFrame,
InputType::kImage};
void RunTest(cv::Mat input, cv::Mat expected_result, float range_min,
float range_max, int tensor_width, int tensor_height,
bool keep_aspect, absl::optional<BorderMode> border_mode,
void RunTest(cv::Mat input, cv::Mat expected_result,
std::vector<float> float_range, std::vector<int> int_range,
int tensor_width, int tensor_height, bool keep_aspect,
absl::optional<BorderMode> border_mode,
const mediapipe::NormalizedRect& roi) {
ASSERT_EQ(2, float_range.size());
ASSERT_EQ(2, int_range.size());
for (auto input_type : kInputTypesToTest) {
RunTestWithInputImagePacket(
input_type == InputType::kImageFrame ? MakeImageFramePacket(input)
: MakeImagePacket(input),
expected_result, range_min, range_max, tensor_width, tensor_height,
keep_aspect, border_mode, roi);
expected_result, float_range[0], float_range[1], tensor_width,
tensor_height, keep_aspect, border_mode, roi,
/*output_int_tensor=*/false);
RunTestWithInputImagePacket(
input_type == InputType::kImageFrame ? MakeImageFramePacket(input)
: MakeImagePacket(input),
expected_result, int_range[0], int_range[1], tensor_width,
tensor_height, keep_aspect, border_mode, roi,
/*output_int_tensor=*/true);
}
}
@@ -195,8 +224,8 @@ TEST(ImageToTensorCalculatorTest, MediumSubRectKeepAspect) {
"tensor/testdata/image_to_tensor/input.jpg"),
GetRgb("/mediapipe/calculators/"
"tensor/testdata/image_to_tensor/medium_sub_rect_keep_aspect.png"),
/*range_min=*/0.0f,
/*range_max=*/1.0f,
/*float_range=*/{0.0f, 1.0f},
/*int_range=*/{0, 255},
/*tensor_width=*/256, /*tensor_height=*/256, /*keep_aspect=*/true,
/*border mode*/ {}, roi);
}
@@ -213,8 +242,8 @@ TEST(ImageToTensorCalculatorTest, MediumSubRectKeepAspectBorderZero) {
GetRgb("/mediapipe/calculators/"
"tensor/testdata/image_to_tensor/"
"medium_sub_rect_keep_aspect_border_zero.png"),
/*range_min=*/0.0f,
/*range_max=*/1.0f,
/*float_range=*/{0.0f, 1.0f},
/*int_range=*/{0, 255},
/*tensor_width=*/256, /*tensor_height=*/256, /*keep_aspect=*/true,
BorderMode::kZero, roi);
}
@@ -231,7 +260,8 @@ TEST(ImageToTensorCalculatorTest, MediumSubRectKeepAspectWithRotation) {
GetRgb("/mediapipe/calculators/"
"tensor/testdata/image_to_tensor/"
"medium_sub_rect_keep_aspect_with_rotation.png"),
/*range_min=*/0.0f, /*range_max=*/1.0f,
/*float_range=*/{0.0f, 1.0f},
/*int_range=*/{0, 255},
/*tensor_width=*/256, /*tensor_height=*/256, /*keep_aspect=*/true,
BorderMode::kReplicate, roi);
}
@@ -249,7 +279,8 @@ TEST(ImageToTensorCalculatorTest,
GetRgb("/mediapipe/calculators/"
"tensor/testdata/image_to_tensor/"
"medium_sub_rect_keep_aspect_with_rotation_border_zero.png"),
/*range_min=*/0.0f, /*range_max=*/1.0f,
/*float_range=*/{0.0f, 1.0f},
/*int_range=*/{0, 255},
/*tensor_width=*/256, /*tensor_height=*/256, /*keep_aspect=*/true,
BorderMode::kZero, roi);
}
@@ -267,8 +298,8 @@ TEST(ImageToTensorCalculatorTest, MediumSubRectWithRotation) {
GetRgb(
"/mediapipe/calculators/"
"tensor/testdata/image_to_tensor/medium_sub_rect_with_rotation.png"),
/*range_min=*/-1.0f,
/*range_max=*/1.0f,
/*float_range=*/{-1.0f, 1.0f},
/*int_range=*/{0, 255},
/*tensor_width=*/256, /*tensor_height=*/256, /*keep_aspect=*/false,
BorderMode::kReplicate, roi);
}
@@ -285,8 +316,8 @@ TEST(ImageToTensorCalculatorTest, MediumSubRectWithRotationBorderZero) {
GetRgb("/mediapipe/calculators/"
"tensor/testdata/image_to_tensor/"
"medium_sub_rect_with_rotation_border_zero.png"),
/*range_min=*/-1.0f,
/*range_max=*/1.0f,
/*float_range=*/{-1.0f, 1.0f},
/*int_range=*/{0, 255},
/*tensor_width=*/256, /*tensor_height=*/256, /*keep_aspect=*/false,
BorderMode::kZero, roi);
}
@@ -302,8 +333,8 @@ TEST(ImageToTensorCalculatorTest, LargeSubRect) {
"tensor/testdata/image_to_tensor/input.jpg"),
GetRgb("/mediapipe/calculators/"
"tensor/testdata/image_to_tensor/large_sub_rect.png"),
/*range_min=*/0.0f,
/*range_max=*/1.0f,
/*float_range=*/{0.0f, 1.0f},
/*int_range=*/{0, 255},
/*tensor_width=*/128, /*tensor_height=*/128, /*keep_aspect=*/false,
BorderMode::kReplicate, roi);
}
@@ -320,8 +351,8 @@ TEST(ImageToTensorCalculatorTest, LargeSubRectBorderZero) {
"tensor/testdata/image_to_tensor/input.jpg"),
GetRgb("/mediapipe/calculators/"
"tensor/testdata/image_to_tensor/large_sub_rect_border_zero.png"),
/*range_min=*/0.0f,
/*range_max=*/1.0f,
/*float_range=*/{0.0f, 1.0f},
/*int_range=*/{0, 255},
/*tensor_width=*/128, /*tensor_height=*/128, /*keep_aspect=*/false,
BorderMode::kZero, roi);
}
@@ -338,8 +369,8 @@ TEST(ImageToTensorCalculatorTest, LargeSubRectKeepAspect) {
"tensor/testdata/image_to_tensor/input.jpg"),
GetRgb("/mediapipe/calculators/"
"tensor/testdata/image_to_tensor/large_sub_rect_keep_aspect.png"),
/*range_min=*/0.0f,
/*range_max=*/1.0f,
/*float_range=*/{0.0f, 1.0f},
/*int_range=*/{0, 255},
/*tensor_width=*/128, /*tensor_height=*/128, /*keep_aspect=*/true,
BorderMode::kReplicate, roi);
}
@@ -356,8 +387,8 @@ TEST(ImageToTensorCalculatorTest, LargeSubRectKeepAspectBorderZero) {
GetRgb("/mediapipe/calculators/"
"tensor/testdata/image_to_tensor/"
"large_sub_rect_keep_aspect_border_zero.png"),
/*range_min=*/0.0f,
/*range_max=*/1.0f,
/*float_range=*/{0.0f, 1.0f},
/*int_range=*/{0, 255},
/*tensor_width=*/128, /*tensor_height=*/128, /*keep_aspect=*/true,
BorderMode::kZero, roi);
}
@@ -374,8 +405,8 @@ TEST(ImageToTensorCalculatorTest, LargeSubRectKeepAspectWithRotation) {
GetRgb("/mediapipe/calculators/"
"tensor/testdata/image_to_tensor/"
"large_sub_rect_keep_aspect_with_rotation.png"),
/*range_min=*/0.0f,
/*range_max=*/1.0f,
/*float_range=*/{0.0f, 1.0f},
/*int_range=*/{0, 255},
/*tensor_width=*/128, /*tensor_height=*/128, /*keep_aspect=*/true,
/*border_mode=*/{}, roi);
}
@@ -393,8 +424,8 @@ TEST(ImageToTensorCalculatorTest,
GetRgb("/mediapipe/calculators/"
"tensor/testdata/image_to_tensor/"
"large_sub_rect_keep_aspect_with_rotation_border_zero.png"),
/*range_min=*/0.0f,
/*range_max=*/1.0f,
/*float_range=*/{0.0f, 1.0f},
/*int_range=*/{0, 255},
/*tensor_width=*/128, /*tensor_height=*/128, /*keep_aspect=*/true,
/*border_mode=*/BorderMode::kZero, roi);
}
@@ -410,8 +441,8 @@ TEST(ImageToTensorCalculatorTest, NoOpExceptRange) {
"tensor/testdata/image_to_tensor/input.jpg"),
GetRgb("/mediapipe/calculators/"
"tensor/testdata/image_to_tensor/noop_except_range.png"),
/*range_min=*/0.0f,
/*range_max=*/1.0f,
/*float_range=*/{0.0f, 1.0f},
/*int_range=*/{0, 255},
/*tensor_width=*/64, /*tensor_height=*/128, /*keep_aspect=*/true,
BorderMode::kReplicate, roi);
}
@@ -427,8 +458,8 @@ TEST(ImageToTensorCalculatorTest, NoOpExceptRangeBorderZero) {
"tensor/testdata/image_to_tensor/input.jpg"),
GetRgb("/mediapipe/calculators/"
"tensor/testdata/image_to_tensor/noop_except_range.png"),
/*range_min=*/0.0f,
/*range_max=*/1.0f,
/*float_range=*/{0.0f, 1.0f},
/*int_range=*/{0, 255},
/*tensor_width=*/64, /*tensor_height=*/128, /*keep_aspect=*/true,
BorderMode::kZero, roi);
}
mediapipe/calculators/tensor/image_to_tensor_converter_gl_texture.cc
+2 -2
View File
@@ -16,7 +16,7 @@
#include "mediapipe/framework/port.h"
#if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_20
#if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_30
#include <array>
#include <memory>
@@ -339,4 +339,4 @@ CreateImageToGlTextureTensorConverter(CalculatorContext* cc,
} // namespace mediapipe
#endif // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_20
#endif // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_30
mediapipe/calculators/tensor/image_to_tensor_converter_gl_texture.h
+2 -2
View File
@@ -17,7 +17,7 @@
#include "mediapipe/framework/port.h"
#if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_20
#if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_30
#include <memory>
@@ -37,6 +37,6 @@ CreateImageToGlTextureTensorConverter(CalculatorContext* cc,
} // namespace mediapipe
#endif // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_20
#endif // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_30
#endif // MEDIAPIPE_CALCULATORS_TENSOR_IMAGE_TO_TENSOR_CONVERTER_GL_TEXTURE_H_
mediapipe/calculators/tensor/image_to_tensor_converter_gl_utils.cc
+2 -2
View File
@@ -2,7 +2,7 @@
#include "mediapipe/framework/port.h"
#if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_20
#if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_30
#include <array>
#include <memory>
@@ -85,4 +85,4 @@ bool IsGlClampToBorderSupported(const mediapipe::GlContext& gl_context) {
} // namespace mediapipe
#endif // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_20
#endif // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_30
mediapipe/calculators/tensor/image_to_tensor_converter_gl_utils.h
+2 -2
View File
@@ -3,7 +3,7 @@
#include "mediapipe/framework/port.h"
#if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_20
#if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_30
#include <array>
#include <memory>
@@ -40,6 +40,6 @@ bool IsGlClampToBorderSupported(const mediapipe::GlContext& gl_context);
} // namespace mediapipe
#endif // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_20
#endif // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_30
#endif // MEDIAPIPE_CALCULATORS_TENSOR_IMAGE_TO_TENSOR_CONVERTER_GL_UTILS_H_
mediapipe/calculators/tensor/image_to_tensor_converter_gl_utils_test.cc
+2 -2
View File
@@ -1,6 +1,6 @@
#include "mediapipe/framework/port.h"
#if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_20
#if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_30
#include "mediapipe/calculators/tensor/image_to_tensor_converter_gl_utils.h"
#include "mediapipe/framework/port/gtest.h"
@@ -46,4 +46,4 @@ TEST(ImageToTensorConverterGlUtilsTest, GlTexParameteriOverrider) {
} // namespace
} // namespace mediapipe
#endif // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_20
#endif // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_30
mediapipe/calculators/tensor/image_to_tensor_converter_opencv.cc
+21 -11
View File
@@ -35,7 +35,8 @@ namespace {
class OpenCvProcessor : public ImageToTensorConverter {
public:
OpenCvProcessor(BorderMode border_mode) {
OpenCvProcessor(BorderMode border_mode, Tensor::ElementType tensor_type)
: tensor_type_(tensor_type) {
switch (border_mode) {
case BorderMode::kReplicate:
border_mode_ = cv::BORDER_REPLICATE;
@@ -44,6 +45,7 @@ class OpenCvProcessor : public ImageToTensorConverter {
border_mode_ = cv::BORDER_CONSTANT;
break;
}
mat_type_ = tensor_type == Tensor::ElementType::kUInt8 ? CV_8UC3 : CV_32FC3;
}
absl::StatusOr<Tensor> Convert(const mediapipe::Image& input,
@@ -56,15 +58,20 @@ class OpenCvProcessor : public ImageToTensorConverter {
absl::StrCat("Only RGBA/RGB formats are supported, passed format: ",
static_cast<uint32_t>(input.image_format())));
}
cv::Mat src = mediapipe::formats::MatView(&input);
auto src = mediapipe::formats::MatView(&input);
constexpr int kNumChannels = 3;
Tensor tensor(
Tensor::ElementType::kFloat32,
Tensor::Shape{1, output_dims.height, output_dims.width, kNumChannels});
Tensor tensor(tensor_type_, Tensor::Shape{1, output_dims.height,
output_dims.width, kNumChannels});
auto buffer_view = tensor.GetCpuWriteView();
cv::Mat dst(output_dims.height, output_dims.width, CV_32FC3,
buffer_view.buffer<float>());
cv::Mat dst;
if (tensor_type_ == Tensor::ElementType::kUInt8) {
dst = cv::Mat(output_dims.height, output_dims.width, mat_type_,
buffer_view.buffer<uint8>());
} else {
dst = cv::Mat(output_dims.height, output_dims.width, mat_type_,
buffer_view.buffer<float>());
}
const cv::RotatedRect rotated_rect(cv::Point2f(roi.center_x, roi.center_y),
cv::Size2f(roi.width, roi.height),
@@ -85,7 +92,7 @@ class OpenCvProcessor : public ImageToTensorConverter {
cv::Mat projection_matrix =
cv::getPerspectiveTransform(src_points, dst_points);
cv::Mat transformed;
cv::warpPerspective(src, transformed, projection_matrix,
cv::warpPerspective(*src, transformed, projection_matrix,
cv::Size(dst_width, dst_height),
/*flags=*/cv::INTER_LINEAR,
/*borderMode=*/border_mode_);
@@ -102,19 +109,22 @@ class OpenCvProcessor : public ImageToTensorConverter {
auto transform,
GetValueRangeTransformation(kInputImageRangeMin, kInputImageRangeMax,
range_min, range_max));
transformed.convertTo(dst, CV_32FC3, transform.scale, transform.offset);
transformed.convertTo(dst, mat_type_, transform.scale, transform.offset);
return tensor;
}
private:
enum cv::BorderTypes border_mode_;
Tensor::ElementType tensor_type_;
int mat_type_;
};
} // namespace
absl::StatusOr<std::unique_ptr<ImageToTensorConverter>> CreateOpenCvConverter(
CalculatorContext* cc, BorderMode border_mode) {
return absl::make_unique<OpenCvProcessor>(border_mode);
CalculatorContext* cc, BorderMode border_mode,
Tensor::ElementType tensor_type) {
return absl::make_unique<OpenCvProcessor>(border_mode, tensor_type);
}
} // namespace mediapipe
mediapipe/calculators/tensor/image_to_tensor_converter_opencv.h
+2 -1
View File
@@ -25,7 +25,8 @@ namespace mediapipe {
// Creates OpenCV image-to-tensor converter.
absl::StatusOr<std::unique_ptr<ImageToTensorConverter>> CreateOpenCvConverter(
CalculatorContext* cc, BorderMode border_mode);
CalculatorContext* cc, BorderMode border_mode,
Tensor::ElementType tensor_type);
} // namespace mediapipe
mediapipe/calculators/tensor/inference_calculator.cc
+1
View File
@@ -36,6 +36,7 @@ class InferenceCalculatorSelectorImpl
Subgraph::GetOptions<mediapipe::InferenceCalculatorOptions>(
subgraph_node);
std::vector<absl::string_view> impls;
const bool should_use_gpu =
!options.has_delegate() || // Use GPU delegate if not specified
(options.has_delegate() && options.delegate().has_gpu());
mediapipe/calculators/tensor/inference_calculator_cpu.cc
+17 -9
View File
@@ -81,6 +81,7 @@ class InferenceCalculatorCpuImpl
Packet<TfLiteModelPtr> model_packet_;
std::unique_ptr<tflite::Interpreter> interpreter_;
TfLiteDelegatePtr delegate_;
bool has_quantized_input_;
};
absl::Status InferenceCalculatorCpuImpl::UpdateContract(
@@ -109,10 +110,18 @@ absl::Status InferenceCalculatorCpuImpl::Process(CalculatorContext* cc) {
for (int i = 0; i < input_tensors.size(); ++i) {
const Tensor* input_tensor = &input_tensors[i];
auto input_tensor_view = input_tensor->GetCpuReadView();
auto input_tensor_buffer = input_tensor_view.buffer<float>();
float* local_tensor_buffer = interpreter_->typed_input_tensor<float>(i);
std::memcpy(local_tensor_buffer, input_tensor_buffer,
input_tensor->bytes());
if (has_quantized_input_) {
// TODO: Support more quantized tensor types.
auto input_tensor_buffer = input_tensor_view.buffer<uint8>();
uint8* local_tensor_buffer = interpreter_->typed_input_tensor<uint8>(i);
std::memcpy(local_tensor_buffer, input_tensor_buffer,
input_tensor->bytes());
} else {
auto input_tensor_buffer = input_tensor_view.buffer<float>();
float* local_tensor_buffer = interpreter_->typed_input_tensor<float>(i);
std::memcpy(local_tensor_buffer, input_tensor_buffer,
input_tensor->bytes());
}
}
// Run inference.
@@ -167,10 +176,9 @@ absl::Status InferenceCalculatorCpuImpl::LoadDelegateAndAllocateTensors(
// AllocateTensors() can be called only after ModifyGraphWithDelegate.
RET_CHECK_EQ(interpreter_->AllocateTensors(), kTfLiteOk);
// TODO: Support quantized tensors.
RET_CHECK_NE(
interpreter_->tensor(interpreter_->inputs()[0])->quantization.type,
kTfLiteAffineQuantization);
has_quantized_input_ =
interpreter_->tensor(interpreter_->inputs()[0])->quantization.type ==
kTfLiteAffineQuantization;
return absl::OkStatus();
}
@@ -226,7 +234,7 @@ absl::Status InferenceCalculatorCpuImpl::LoadDelegate(CalculatorContext* cc) {
#endif // defined(__EMSCRIPTEN__)
if (use_xnnpack) {
TfLiteXNNPackDelegateOptions xnnpack_opts{};
auto xnnpack_opts = TfLiteXNNPackDelegateOptionsDefault();
xnnpack_opts.num_threads =
GetXnnpackNumThreads(opts_has_delegate, opts_delegate);
delegate_ = TfLiteDelegatePtr(TfLiteXNNPackDelegateCreate(&xnnpack_opts),
mediapipe/calculators/tensor/inference_calculator_face_detection_test.cc
+3 -2
View File
@@ -154,8 +154,9 @@ TEST_P(InferenceCalculatorTest, TestFaceDetection) {
detection_packets[0].Get<std::vector<Detection>>();
#if !defined(MEDIAPIPE_PROTO_LITE)
// Approximately is not available with lite protos (b/178137094).
EXPECT_THAT(dets,
ElementsAre(Approximately(EqualsProto(expected_detection))));
constexpr float kEpison = 0.001;
EXPECT_THAT(dets, ElementsAre(Approximately(EqualsProto(expected_detection),
kEpison)));
#endif
}
mediapipe/calculators/tensor/inference_calculator_gl.cc
+10 -4
View File
@@ -59,8 +59,6 @@ class InferenceCalculatorGlImpl
// TfLite requires us to keep the model alive as long as the interpreter is.
Packet<TfLiteModelPtr> model_packet_;
std::unique_ptr<tflite::Interpreter> interpreter_;
TfLiteDelegatePtr delegate_;
#if MEDIAPIPE_TFLITE_GL_INFERENCE
mediapipe::GlCalculatorHelper gpu_helper_;
@@ -72,6 +70,9 @@ class InferenceCalculatorGlImpl
tflite_gpu_runner_usage_;
#endif // MEDIAPIPE_TFLITE_GL_INFERENCE
TfLiteDelegatePtr delegate_;
std::unique_ptr<tflite::Interpreter> interpreter_;
#if MEDIAPIPE_TFLITE_GPU_SUPPORTED
std::vector<Tensor::Shape> output_shapes_;
std::vector<std::unique_ptr<Tensor>> gpu_buffers_in_;
@@ -252,12 +253,17 @@ absl::Status InferenceCalculatorGlImpl::Close(CalculatorContext* cc) {
MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext([this]() -> Status {
gpu_buffers_in_.clear();
gpu_buffers_out_.clear();
// Delegate must outlive the interpreter, hence the order is important.
interpreter_ = nullptr;
delegate_ = nullptr;
return absl::OkStatus();
}));
} else {
// Delegate must outlive the interpreter, hence the order is important.
interpreter_ = nullptr;
delegate_ = nullptr;
}
interpreter_ = nullptr;
delegate_ = nullptr;
return absl::OkStatus();
}
mediapipe/calculators/tensor/tensors_to_detections_calculator.cc
+2 -1
View File
@@ -266,6 +266,7 @@ absl::Status TensorsToDetectionsCalculator::ProcessCPU(
auto raw_box_tensor = &input_tensors[0];
RET_CHECK_EQ(raw_box_tensor->shape().dims.size(), 3);
RET_CHECK_EQ(raw_box_tensor->shape().dims[0], 1);
RET_CHECK_GT(num_boxes_, 0) << "Please set num_boxes in calculator options";
RET_CHECK_EQ(raw_box_tensor->shape().dims[1], num_boxes_);
RET_CHECK_EQ(raw_box_tensor->shape().dims[2], num_coords_);
auto raw_score_tensor = &input_tensors[1];
@@ -385,6 +386,7 @@ absl::Status TensorsToDetectionsCalculator::ProcessGPU(
CalculatorContext* cc, std::vector<Detection>* output_detections) {
const auto& input_tensors = *kInTensors(cc);
RET_CHECK_GE(input_tensors.size(), 2);
RET_CHECK_GT(num_boxes_, 0) << "Please set num_boxes in calculator options";
#ifndef MEDIAPIPE_DISABLE_GL_COMPUTE
MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext([this, &input_tensors, &cc,
@@ -563,7 +565,6 @@ absl::Status TensorsToDetectionsCalculator::LoadOptions(CalculatorContext* cc) {
// Get calculator options specified in the graph.
options_ = cc->Options<::mediapipe::TensorsToDetectionsCalculatorOptions>();
RET_CHECK(options_.has_num_classes());
RET_CHECK(options_.has_num_boxes());
RET_CHECK(options_.has_num_coords());
num_classes_ = options_.num_classes();
mediapipe/calculators/tensor/tensors_to_segmentation_calculator.cc
+3 -2
View File
@@ -355,9 +355,10 @@ absl::Status TensorsToSegmentationCalculator::ProcessCpu(
std::shared_ptr<ImageFrame> mask_frame = std::make_shared<ImageFrame>(
ImageFormat::VEC32F1, output_width, output_height);
std::unique_ptr<Image> output_mask = absl::make_unique<Image>(mask_frame);
cv::Mat output_mat = formats::MatView(output_mask.get());
auto output_mat = formats::MatView(output_mask.get());
// Upsample small mask into output.
cv::resize(small_mask_mat, output_mat, cv::Size(output_width, output_height));
cv::resize(small_mask_mat, *output_mat,
cv::Size(output_width, output_height));
cc->Outputs().Tag(kMaskTag).Add(output_mask.release(), cc->InputTimestamp());
return absl::OkStatus();