talksik/mediapipe
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Project import generated by Copybara.

Browse Source 
GitOrigin-RevId: 1e13be30e2c6838d4a2ff768a39c414bc80534bb
This commit is contained in:
MediaPipe Team
2022-09-06 21:46:17 +00:00
committed by Sebastian Schmidt
parent 63e679d99c
commit 4dc4b19ddb
639 changed files with 71327 additions and 2078 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mediapipe/calculators/tensor/BUILD
+60 -1
View File
@@ -153,11 +153,12 @@ cc_library(
tags = ["nomac"], # config problem with cpuinfo via TF
visibility = ["//visibility:public"],
deps = [
":inference_calculator_cc_proto",
":inference_calculator_interface",
"//mediapipe/framework:calculator_context",
"//mediapipe/gpu:gl_calculator_helper",
"@com_google_absl//absl/memory",
"@com_google_absl//absl/status",
"@org_tensorflow//tensorflow/lite:framework_stable",
"@org_tensorflow//tensorflow/lite/delegates/gpu:gl_delegate",
],
alwayslink = 1,
@@ -172,6 +173,7 @@ cc_library(
":inference_calculator_interface",
"@com_google_absl//absl/memory",
"@com_google_absl//absl/status",
"@com_google_absl//absl/status:statusor",
"//mediapipe/framework/deps:file_path",
"//mediapipe/gpu:gl_calculator_helper",
"//mediapipe/util/tflite:tflite_gpu_runner",
@@ -231,6 +233,7 @@ cc_library(
deps = [
":inference_calculator_interface",
"@com_google_absl//absl/memory",
"@com_google_absl//absl/status",
"@org_tensorflow//tensorflow/lite/delegates/xnnpack:xnnpack_delegate",
"@org_tensorflow//tensorflow/lite:framework_stable",
"@org_tensorflow//tensorflow/lite/c:c_api_types",
@@ -636,6 +639,7 @@ cc_library(
":image_to_tensor_calculator_cc_proto",
":image_to_tensor_converter",
":image_to_tensor_utils",
":loose_headers",
"//mediapipe/framework/api2:node",
"//mediapipe/framework/formats:image",
"//mediapipe/framework/formats:image_frame",
@@ -990,3 +994,58 @@ cc_library(
}),
alwayslink = 1,
)
cc_library(
name = "tensors_dequantization_calculator",
srcs = ["tensors_dequantization_calculator.cc"],
copts = select({
"//mediapipe:apple": [
"-x objective-c++",
"-fobjc-arc", # enable reference-counting
],
"//conditions:default": [],
}),
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_context",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/api2:node",
"//mediapipe/framework/api2:port",
"//mediapipe/framework/formats:tensor",
"//mediapipe/framework/port:ret_check",
"@com_google_absl//absl/status",
"@com_google_absl//absl/strings",
],
alwayslink = 1,
)
# For a more maintainable build this target should not exist and the headers
# should be split into the existing cc_library targets, but this change was
# automatically done so that we can remove long standing issues and complexity
# in the build system. It's up to the OWNERS of this package to get rid of it or
# not. The use of the textual_hdrs attribute is discouraged, use hdrs instead.
# Here it is used to avoid header parsing errors in packages where the feature
# parse_headers was enabled since loose headers were not being parsed.
cc_library(
name = "loose_headers",
tags = ["avoid_dep"],
textual_hdrs = [
"image_to_tensor_converter_gl_buffer.h",
"image_to_tensor_converter_gl_texture.h",
],
visibility = [":__pkg__"],
)
cc_test(
name = "tensors_dequantization_calculator_test",
srcs = ["tensors_dequantization_calculator_test.cc"],
deps = [
":tensors_dequantization_calculator",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:calculator_runner",
"//mediapipe/framework/formats:tensor",
"//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:parse_text_proto",
"@com_google_absl//absl/status",
],
)
mediapipe/calculators/tensor/audio_to_tensor_calculator.cc
+30 -22
View File
@@ -56,14 +56,14 @@ namespace api2 {
// previous output.
//
// The calculator has two running modes:
// Streaming mode: when "streaming_mode" is set to true in the calculator
// Streaming mode: when "stream_mode" is set to true in the calculator
// options, the calculator treats the input audio stream as a continuous
// stream. Thus, any samples that are not consumed in the previous runs will
// be cached in a global sample buffer. The audio data resampled from the
// current raw audio input will be appended to the global sample buffer.
// The calculator will process the global sample buffer and output as many
// tensors as possible.
// Non-streaming mode: when "streaming_mode" is set to false in the calculator
// Non-streaming mode: when "stream_mode" is set to false in the calculator
// options, the calculators treats the packets in the input audio stream as
// a batch of unrelated audio buffers. In each Process() call, the input
// buffer will be frist resampled, and framed as fixed-sized, possibly
@@ -104,7 +104,7 @@ namespace api2 {
// num_samples: 512
// num_overlapping_samples: 64
// target_sample_rate: 16000
// streaming_mode: true # or false
// stream_mode: true # or false
// }
// }
// }
@@ -136,7 +136,7 @@ class AudioToTensorCalculator : public Node {
// The number of samples per channel to advance after the current frame is
// processed.
int frame_step_;
bool streaming_mode_;
bool stream_mode_;
bool check_inconsistent_timestamps_;
Timestamp initial_timestamp_ = Timestamp::Unstarted();
int64 cumulative_input_samples_ = 0;
@@ -151,8 +151,9 @@ class AudioToTensorCalculator : public Node {
Matrix sample_buffer_;
int processed_buffer_cols_ = 0;
absl::Status ProcessStreamingData(CalculatorContext* cc);
absl::Status ProcessNonStreamingData(CalculatorContext* cc);
absl::Status ProcessStreamingData(CalculatorContext* cc, const Matrix& input);
absl::Status ProcessNonStreamingData(CalculatorContext* cc,
const Matrix& input);
absl::Status SetupStreamingResampler(double input_sample_rate_);
void AppendToSampleBuffer(Matrix buffer_to_append);
@@ -172,7 +173,7 @@ absl::Status AudioToTensorCalculator::UpdateContract(CalculatorContract* cc) {
"AudioToTensorCalculatorOptions must specifiy "
"`num_channels`, `num_samples`, and `target_sample_rate`.");
}
if (options.streaming_mode()) {
if (options.stream_mode()) {
// Explicitly disables tiemstamp offset to disallow the timestamp bound
// from the input streams to be propagated to the output streams.
// In the streaming mode, the output timestamp bound is based on
@@ -196,8 +197,8 @@ absl::Status AudioToTensorCalculator::Open(CalculatorContext* cc) {
frame_step_ = num_samples_;
}
target_sample_rate_ = options.target_sample_rate();
streaming_mode_ = options.streaming_mode();
if (streaming_mode_) {
stream_mode_ = options.stream_mode();
if (stream_mode_) {
check_inconsistent_timestamps_ = options.check_inconsistent_timestamps();
sample_buffer_.resize(num_channels_, Eigen::NoChange);
}
@@ -210,7 +211,7 @@ absl::Status AudioToTensorCalculator::Open(CalculatorContext* cc) {
mediapipe::TimeSeriesHeader input_header;
MP_RETURN_IF_ERROR(mediapipe::time_series_util::FillTimeSeriesHeaderIfValid(
kAudioIn(cc).Header(), &input_header));
if (streaming_mode_) {
if (stream_mode_) {
MP_RETURN_IF_ERROR(SetupStreamingResampler(input_header.sample_rate()));
} else {
source_sample_rate_ = input_header.sample_rate();
@@ -223,7 +224,7 @@ absl::Status AudioToTensorCalculator::Process(CalculatorContext* cc) {
if (cc->InputTimestamp() == Timestamp::PreStream()) {
double current_source_sample_rate = kAudioSampleRateIn(cc).Get();
if (cc->Options<mediapipe::AudioToTensorCalculatorOptions>()
.streaming_mode()) {
.stream_mode()) {
return SetupStreamingResampler(current_source_sample_rate);
} else {
source_sample_rate_ = current_source_sample_rate;
@@ -232,21 +233,28 @@ absl::Status AudioToTensorCalculator::Process(CalculatorContext* cc) {
}
// Sanity checks.
const auto& input_frame = kAudioIn(cc).Get();
if (input_frame.rows() != num_channels_) {
const bool channels_match = input_frame.rows() == num_channels_;
// The special case of `num_channels_ == 1` is automatic mixdown to mono.
const bool mono_output = num_channels_ == 1;
if (!mono_output && !channels_match) {
return absl::InvalidArgumentError(absl::StrFormat(
"Audio input has %d channel(s) but the model requires %d channel(s).",
input_frame.rows(), num_channels_));
}
if (num_channels_ > 1 && input_frame.IsRowMajor) {
if (!mono_output && input_frame.IsRowMajor) {
return absl::InvalidArgumentError(
"The audio data should be stored in column-major.");
}
return streaming_mode_ ? ProcessStreamingData(cc)
: ProcessNonStreamingData(cc);
CHECK(channels_match || mono_output);
const Matrix& input = channels_match ? input_frame
// Mono mixdown.
: input_frame.colwise().mean();
return stream_mode_ ? ProcessStreamingData(cc, input)
: ProcessNonStreamingData(cc, input);
}
absl::Status AudioToTensorCalculator::Close(CalculatorContext* cc) {
if (!streaming_mode_) {
if (!stream_mode_) {
return absl::OkStatus();
}
if (resampler_) {
@@ -258,8 +266,8 @@ absl::Status AudioToTensorCalculator::Close(CalculatorContext* cc) {
}
absl::Status AudioToTensorCalculator::ProcessStreamingData(
CalculatorContext* cc) {
const auto& input_buffer = kAudioIn(cc).Get();
CalculatorContext* cc, const Matrix& input) {
const auto& input_buffer = input;
if (initial_timestamp_ == Timestamp::Unstarted()) {
initial_timestamp_ = cc->InputTimestamp();
next_output_timestamp_ = initial_timestamp_;
@@ -303,10 +311,10 @@ absl::Status AudioToTensorCalculator::ProcessStreamingData(
}
absl::Status AudioToTensorCalculator::ProcessNonStreamingData(
CalculatorContext* cc) {
CalculatorContext* cc, const Matrix& input) {
initial_timestamp_ = cc->InputTimestamp();
next_output_timestamp_ = initial_timestamp_;
const auto& input_frame = kAudioIn(cc).Get();
const auto& input_frame = input;
double source_sample_rate = kAudioSampleRateIn(cc).GetOr(source_sample_rate_);
if (source_sample_rate != -1 && source_sample_rate != target_sample_rate_) {
@@ -362,7 +370,7 @@ absl::Status AudioToTensorCalculator::OutputTensors(const Matrix& buffer,
CalculatorContext* cc) {
int next_frame_first_col = 0;
std::vector<Timestamp> timestamps;
while ((!streaming_mode_ || !should_flush) &&
while ((!stream_mode_ || !should_flush) &&
next_frame_first_col + num_samples_ <= buffer.cols()) {
ASSIGN_OR_RETURN(auto output_tensor, ConvertToTensor(buffer.block(
0, next_frame_first_col,
@@ -383,7 +391,7 @@ absl::Status AudioToTensorCalculator::OutputTensors(const Matrix& buffer,
// Timestamp::Max() will be emitted. In the non-streaming mode, each
// Process() invocation will process the entire buffer completely.
Timestamp timestamp =
streaming_mode_ ? Timestamp::Max() : next_output_timestamp_;
stream_mode_ ? Timestamp::Max() : next_output_timestamp_;
timestamps.push_back(timestamp);
kTensorsOut(cc).Send(std::move(output_tensor), timestamp);
}
mediapipe/calculators/tensor/audio_to_tensor_calculator.proto
+2 -1
View File
@@ -24,6 +24,7 @@ message AudioToTensorCalculatorOptions {
}
// The required number of channels the output audio tensor has.
// If set to 1, multichannel signals will be automatically mixed down to mono.
optional int64 num_channels = 1;
// The required number of samples per channel the output audio tensor has.
@@ -38,7 +39,7 @@ message AudioToTensorCalculatorOptions {
// Whether to treat the input audio stream as a continous stream or a batch
// of unrelated audio buffers.
optional bool streaming_mode = 5 [default = true];
optional bool stream_mode = 5 [default = true];
// Set to false to disable checks for jitter in timestamp values. Useful with
// live audio input.
mediapipe/calculators/tensor/audio_to_tensor_calculator_test.cc
+26 -7
View File
@@ -63,7 +63,10 @@ class AudioToTensorCalculatorNonStreamingModeTest : public ::testing::Test {
protected:
void SetUp() override {}
void Run(int num_samples, int num_overlapping_samples,
double resampling_factor, const Matrix& input_matrix) {
double resampling_factor, const Matrix& input_matrix,
int num_channels_override = 0) {
const int num_channels = num_channels_override == 0 ? input_matrix.rows()
: num_channels_override;
double input_sample_rate = 10000;
double target_sample_rate = input_sample_rate * resampling_factor;
auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(
@@ -84,12 +87,12 @@ class AudioToTensorCalculatorNonStreamingModeTest : public ::testing::Test {
num_samples: $1
num_overlapping_samples: $2
target_sample_rate: $3
streaming_mode: false
stream_mode: false
}
}
}
)",
/*$0=*/input_matrix.rows(),
/*$0=*/num_channels,
/*$1=*/num_samples, /*$2=*/num_overlapping_samples,
/*$3=*/target_sample_rate));
tool::AddVectorSink("tensors", &graph_config, &tensors_packets_);
@@ -114,20 +117,21 @@ class AudioToTensorCalculatorNonStreamingModeTest : public ::testing::Test {
}
void CheckTensorsOutputPackets(const Matrix& expected_matrix,
int sample_offset, int num_tensors_per_input) {
int sample_offset, int num_tensors_per_input,
bool mono = false) {
ASSERT_EQ(num_iterations_ * num_tensors_per_input, tensors_packets_.size());
for (int i = 0; i < num_iterations_; ++i) {
for (int j = 0; j < num_tensors_per_input; ++j) {
CheckTensorsOutputPacket(
expected_matrix, tensors_packets_[i * num_tensors_per_input + j],
/*sample_offset*/ sample_offset * j, /*index=*/j);
/*sample_offset=*/sample_offset * j, /*index=*/j, /*mono=*/mono);
}
}
}
void CheckTensorsOutputPacket(const Matrix& expected_matrix,
const Packet& packet, int sample_offset,
int index) {
int index, bool mono = false) {
MP_ASSERT_OK(packet.ValidateAsType<std::vector<Tensor>>());
ASSERT_EQ(1, packet.Get<std::vector<Tensor>>().size());
const Tensor& output_tensor = packet.Get<std::vector<Tensor>>()[0];
@@ -137,7 +141,11 @@ class AudioToTensorCalculatorNonStreamingModeTest : public ::testing::Test {
for (int i = 0; i < num_values; ++i) {
if (i + sample_offset >= expected_matrix.size()) {
EXPECT_FLOAT_EQ(output_floats[i], 0);
} else if (mono) {
EXPECT_FLOAT_EQ(output_floats[i],
expected_matrix.coeff(0, i + sample_offset));
} else {
// Stereo.
EXPECT_FLOAT_EQ(output_floats[i],
expected_matrix.coeff((i + sample_offset) % 2,
(i + sample_offset) / 2))
@@ -209,6 +217,17 @@ TEST_F(AudioToTensorCalculatorNonStreamingModeTest, TensorsWithZeroPadding) {
CloseGraph();
}
TEST_F(AudioToTensorCalculatorNonStreamingModeTest, Mixdown) {
auto input_matrix = CreateTestMatrix(2, 8, 0);
Run(/*num_samples=*/4, /*num_overlapping_samples=*/2,
/*resampling_factor=*/1.0f, *input_matrix, /*num_channels_override=*/1);
const Matrix& mono_matrix = input_matrix->colwise().mean();
CheckTensorsOutputPackets(mono_matrix, /*sample_offset=*/2,
/*num_tensors_per_input=*/4, /*mono=*/true);
CheckTimestampsOutputPackets({0, 200, 400, 600});
CloseGraph();
}
TEST_F(AudioToTensorCalculatorNonStreamingModeTest, Downsampling) {
auto input_matrix = CreateTestMatrix(2, 1024, 0);
Run(/*num_samples=*/256, /*num_overlapping_samples=*/0,
@@ -299,7 +318,7 @@ class AudioToTensorCalculatorStreamingModeTest : public ::testing::Test {
num_samples: $0
num_overlapping_samples: $1
target_sample_rate: $2
streaming_mode:true
stream_mode:true
}
}
}
mediapipe/calculators/tensor/image_to_tensor_calculator.cc
+3 -4
View File
@@ -348,14 +348,13 @@ class ImageToTensorCalculator : public Node {
CreateImageToGlBufferTensorConverter(
cc, DoesGpuInputStartAtBottom(), GetBorderMode()));
#else
// Check whether the underlying storage object is a GL texture.
if (image.GetGpuBuffer()
.internal_storage<mediapipe::GlTextureBuffer>()) {
if (!gpu_converter_) {
ASSIGN_OR_RETURN(
gpu_converter_,
CreateImageToGlTextureTensorConverter(
cc, DoesGpuInputStartAtBottom(), GetBorderMode()));
} else {
}
if (!gpu_converter_) {
return absl::UnimplementedError(
"ImageToTensorConverter for the input GPU image is unavailable.");
}
mediapipe/calculators/tensor/inference_calculator.cc
+1
View File
@@ -19,6 +19,7 @@
#include <string>
#include <vector>
#include "absl/status/status.h"
#include "absl/strings/string_view.h"
#include "mediapipe/calculators/tensor/inference_calculator.pb.h"
#include "mediapipe/framework/api2/packet.h"
mediapipe/calculators/tensor/inference_calculator.h
+2 -7
View File
@@ -20,18 +20,13 @@
#include <string>
#include <vector>
#include "absl/memory/memory.h"
#include "mediapipe/calculators/tensor/inference_calculator.pb.h"
#include "mediapipe/framework/api2/node.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/tensor.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/util/tflite/tflite_model_loader.h"
#include "tensorflow/lite/core/api/op_resolver.h"
#include "tensorflow/lite/error_reporter.h"
#include "tensorflow/lite/interpreter.h"
#include "tensorflow/lite/kernels/register.h"
#include "tensorflow/lite/model.h"
namespace mediapipe {
namespace api2 {
@@ -119,10 +114,10 @@ class InferenceCalculator : public NodeIntf {
using TfLiteDelegatePtr =
std::unique_ptr<TfLiteDelegate, std::function<void(TfLiteDelegate*)>>;
absl::StatusOr<Packet<TfLiteModelPtr>> GetModelAsPacket(
static absl::StatusOr<Packet<TfLiteModelPtr>> GetModelAsPacket(
CalculatorContext* cc);
absl::StatusOr<Packet<tflite::OpResolver>> GetOpResolverAsPacket(
static absl::StatusOr<Packet<tflite::OpResolver>> GetOpResolverAsPacket(
CalculatorContext* cc);
};
mediapipe/calculators/tensor/inference_calculator_cpu.cc
+77 -22
View File
@@ -12,13 +12,16 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include <cstdint>
#include <cstring>
#include <memory>
#include <string>
#include <vector>
#include "absl/memory/memory.h"
#include "absl/status/status.h"
#include "mediapipe/calculators/tensor/inference_calculator.h"
#include "tensorflow/lite/interpreter.h"
#include "tensorflow/lite/interpreter_builder.h"
#if defined(MEDIAPIPE_ANDROID)
#include "tensorflow/lite/delegates/nnapi/nnapi_delegate.h"
@@ -63,9 +66,9 @@ int GetXnnpackNumThreads(
}
template <typename T>
void CopyTensorBuffer(const Tensor& input_tensor,
tflite::Interpreter* interpreter,
int input_tensor_index) {
void CopyTensorBufferToInterpreter(const Tensor& input_tensor,
tflite::Interpreter* interpreter,
int input_tensor_index) {
auto input_tensor_view = input_tensor.GetCpuReadView();
auto input_tensor_buffer = input_tensor_view.buffer<T>();
T* local_tensor_buffer =
@@ -73,6 +76,18 @@ void CopyTensorBuffer(const Tensor& input_tensor,
std::memcpy(local_tensor_buffer, input_tensor_buffer, input_tensor.bytes());
}
template <typename T>
void CopyTensorBufferFromInterpreter(tflite::Interpreter* interpreter,
int output_tensor_index,
Tensor* output_tensor) {
auto output_tensor_view = output_tensor->GetCpuWriteView();
auto output_tensor_buffer = output_tensor_view.buffer<T>();
T* local_tensor_buffer =
interpreter->typed_output_tensor<T>(output_tensor_index);
std::memcpy(output_tensor_buffer, local_tensor_buffer,
output_tensor->bytes());
}
} // namespace
class InferenceCalculatorCpuImpl
@@ -99,7 +114,7 @@ class InferenceCalculatorCpuImpl
absl::Status InferenceCalculatorCpuImpl::UpdateContract(
CalculatorContract* cc) {
const auto& options = cc->Options<::mediapipe::InferenceCalculatorOptions>();
const auto& options = cc->Options<mediapipe::InferenceCalculatorOptions>();
RET_CHECK(!options.model_path().empty() ^ kSideInModel(cc).IsConnected())
<< "Either model as side packet or model path in options is required.";
@@ -118,20 +133,32 @@ absl::Status InferenceCalculatorCpuImpl::Process(CalculatorContext* cc) {
RET_CHECK(!input_tensors.empty());
auto output_tensors = absl::make_unique<std::vector<Tensor>>();
if (input_tensor_type_ == kTfLiteNoType) {
input_tensor_type_ = interpreter_->tensor(interpreter_->inputs()[0])->type;
}
// Read CPU input into tensors.
for (int i = 0; i < input_tensors.size(); ++i) {
switch (input_tensor_type_) {
case TfLiteType::kTfLiteFloat16:
case TfLiteType::kTfLiteFloat32: {
CopyTensorBuffer<float>(input_tensors[i], interpreter_.get(), i);
CopyTensorBufferToInterpreter<float>(input_tensors[i],
interpreter_.get(), i);
break;
}
case TfLiteType::kTfLiteUInt8: {
CopyTensorBuffer<uint8>(input_tensors[i], interpreter_.get(), i);
CopyTensorBufferToInterpreter<uint8>(input_tensors[i],
interpreter_.get(), i);
break;
}
case TfLiteType::kTfLiteInt8: {
CopyTensorBuffer<int8>(input_tensors[i], interpreter_.get(), i);
CopyTensorBufferToInterpreter<int8>(input_tensors[i],
interpreter_.get(), i);
break;
}
case TfLiteType::kTfLiteInt32: {
CopyTensorBufferToInterpreter<int32_t>(input_tensors[i],
interpreter_.get(), i);
break;
}
default:
@@ -148,13 +175,41 @@ absl::Status InferenceCalculatorCpuImpl::Process(CalculatorContext* cc) {
output_tensors->reserve(tensor_indexes.size());
for (int i = 0; i < tensor_indexes.size(); ++i) {
TfLiteTensor* tensor = interpreter_->tensor(tensor_indexes[i]);
output_tensors->emplace_back(
Tensor::ElementType::kFloat32,
Tensor::Shape{std::vector<int>{
tensor->dims->data, tensor->dims->data + tensor->dims->size}});
auto cpu_view = output_tensors->back().GetCpuWriteView();
std::memcpy(cpu_view.buffer<float>(), tensor->data.f,
output_tensors->back().bytes());
Tensor::Shape shape{std::vector<int>{
tensor->dims->data, tensor->dims->data + tensor->dims->size}};
switch (tensor->type) {
case TfLiteType::kTfLiteFloat16:
case TfLiteType::kTfLiteFloat32:
output_tensors->emplace_back(Tensor::ElementType::kFloat32, shape);
CopyTensorBufferFromInterpreter<float>(interpreter_.get(), i,
&output_tensors->back());
break;
case TfLiteType::kTfLiteUInt8:
output_tensors->emplace_back(
Tensor::ElementType::kUInt8, shape,
Tensor::QuantizationParameters{tensor->params.scale,
tensor->params.zero_point});
CopyTensorBufferFromInterpreter<uint8>(interpreter_.get(), i,
&output_tensors->back());
break;
case TfLiteType::kTfLiteInt8:
output_tensors->emplace_back(
Tensor::ElementType::kInt8, shape,
Tensor::QuantizationParameters{tensor->params.scale,
tensor->params.zero_point});
CopyTensorBufferFromInterpreter<int8>(interpreter_.get(), i,
&output_tensors->back());
break;
case TfLiteType::kTfLiteInt32:
output_tensors->emplace_back(Tensor::ElementType::kInt32, shape);
CopyTensorBufferFromInterpreter<int32_t>(interpreter_.get(), i,
&output_tensors->back());
break;
default:
return absl::InvalidArgumentError(
absl::StrCat("Unsupported output tensor type:",
TfLiteTypeGetName(tensor->type)));
}
}
kOutTensors(cc).Send(std::move(output_tensors));
return absl::OkStatus();
@@ -188,7 +243,6 @@ absl::Status InferenceCalculatorCpuImpl::InitInterpreter(
absl::Status InferenceCalculatorCpuImpl::AllocateTensors() {
RET_CHECK_EQ(interpreter_->AllocateTensors(), kTfLiteOk);
input_tensor_type_ = interpreter_->tensor(interpreter_->inputs()[0])->type;
return absl::OkStatus();
}
@@ -198,13 +252,14 @@ absl::Status InferenceCalculatorCpuImpl::LoadDelegate(
cc->Options<mediapipe::InferenceCalculatorOptions>();
auto opts_delegate = calculator_opts.delegate();
if (!kDelegate(cc).IsEmpty()) {
mediapipe::InferenceCalculatorOptions::Delegate input_side_packet_delegate =
kDelegate(cc).Get();
CHECK(input_side_packet_delegate.has_tflite() ||
input_side_packet_delegate.has_xnnpack() ||
input_side_packet_delegate.has_nnapi() ||
input_side_packet_delegate.delegate_case() ==
mediapipe::InferenceCalculatorOptions::Delegate::DELEGATE_NOT_SET)
const mediapipe::InferenceCalculatorOptions::Delegate&
input_side_packet_delegate = kDelegate(cc).Get();
RET_CHECK(
input_side_packet_delegate.has_tflite() ||
input_side_packet_delegate.has_xnnpack() ||
input_side_packet_delegate.has_nnapi() ||
input_side_packet_delegate.delegate_case() ==
mediapipe::InferenceCalculatorOptions::Delegate::DELEGATE_NOT_SET)
<< "inference_calculator_cpu only supports delegate input side packet "
<< "for TFLite, XNNPack and Nnapi";
opts_delegate.MergeFrom(input_side_packet_delegate);
mediapipe/calculators/tensor/inference_calculator_gl.cc
+146 -99
View File
@@ -15,11 +15,14 @@
#include <cstring>
#include <memory>
#include <string>
#include <utility>
#include <vector>
#include "absl/memory/memory.h"
#include "absl/status/status.h"
#include "mediapipe/calculators/tensor/inference_calculator.h"
#include "mediapipe/calculators/tensor/inference_calculator.pb.h"
#include "mediapipe/framework/calculator_context.h"
#include "mediapipe/gpu/gl_calculator_helper.h"
#include "tensorflow/lite/delegates/gpu/gl_delegate.h"
@@ -36,111 +39,64 @@ class InferenceCalculatorGlImpl
absl::Status Close(CalculatorContext* cc) override;
private:
absl::Status LoadModel(CalculatorContext* cc);
absl::Status LoadDelegate(CalculatorContext* cc);
absl::Status LoadDelegateAndAllocateTensors(CalculatorContext* cc);
// Helper class that wraps everything related to GPU inference acceleration.
class GpuInferenceRunner {
public:
~GpuInferenceRunner();
// TfLite requires us to keep the model alive as long as the interpreter is.
Packet<TfLiteModelPtr> model_packet_;
absl::Status Init(CalculatorContext* cc,
const mediapipe::InferenceCalculatorOptions::Delegate&
delegate_options);
absl::Status LoadModel(CalculatorContext* cc);
absl::Status LoadDelegate(
CalculatorContext* cc,
const mediapipe::InferenceCalculatorOptions::Delegate&
delegate_options);
absl::Status LoadDelegateAndAllocateTensors(
CalculatorContext* cc,
const mediapipe::InferenceCalculatorOptions::Delegate&
delegate_options);
absl::Status Process(CalculatorContext* cc,
const std::vector<Tensor>& input_tensors,
std::vector<Tensor>& output_tensors);
mediapipe::GlCalculatorHelper gpu_helper_;
bool allow_precision_loss_ = false;
private:
// TfLite requires us to keep the model alive as long as the interpreter is.
Packet<TfLiteModelPtr> model_packet_;
mediapipe::GlCalculatorHelper gpu_helper_;
TfLiteDelegatePtr delegate_;
std::unique_ptr<tflite::Interpreter> interpreter_;
std::vector<std::unique_ptr<Tensor>> gpu_buffers_in_;
std::vector<std::unique_ptr<Tensor>> gpu_buffers_out_;
size_t output_size_ = 0;
};
TfLiteDelegatePtr delegate_;
std::unique_ptr<tflite::Interpreter> interpreter_;
std::vector<Tensor::Shape> output_shapes_;
std::vector<std::unique_ptr<Tensor>> gpu_buffers_in_;
std::vector<std::unique_ptr<Tensor>> gpu_buffers_out_;
std::unique_ptr<GpuInferenceRunner> gpu_inference_runner_;
};
absl::Status InferenceCalculatorGlImpl::UpdateContract(CalculatorContract* cc) {
const auto& options = cc->Options<::mediapipe::InferenceCalculatorOptions>();
RET_CHECK(!options.model_path().empty() ^ kSideInModel(cc).IsConnected())
<< "Either model as side packet or model path in options is required.";
return mediapipe::GlCalculatorHelper::UpdateContract(cc);
}
absl::Status InferenceCalculatorGlImpl::Open(CalculatorContext* cc) {
const auto& options = cc->Options<::mediapipe::InferenceCalculatorOptions>();
mediapipe::InferenceCalculatorOptions::Delegate delegate = options.delegate();
if (!kDelegate(cc).IsEmpty()) {
mediapipe::InferenceCalculatorOptions::Delegate input_side_packet_delegate =
kDelegate(cc).Get();
CHECK(input_side_packet_delegate.has_gpu() ||
input_side_packet_delegate.delegate_case() ==
mediapipe::InferenceCalculatorOptions::Delegate::DELEGATE_NOT_SET)
<< "inference_calculator_gl only supports delegate input side packet "
<< "for Gpu";
delegate.MergeFrom(input_side_packet_delegate);
}
MP_RETURN_IF_ERROR(LoadModel(cc));
MP_RETURN_IF_ERROR(gpu_helper_.Open(cc));
return gpu_helper_.RunInGlContext([this, &cc]() -> ::mediapipe::Status {
return LoadDelegateAndAllocateTensors(cc);
});
}
absl::Status InferenceCalculatorGlImpl::Process(CalculatorContext* cc) {
if (kInTensors(cc).IsEmpty()) {
return absl::OkStatus();
}
const auto& input_tensors = *kInTensors(cc);
RET_CHECK(!input_tensors.empty());
auto output_tensors = absl::make_unique<std::vector<Tensor>>();
MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext(
[this, &input_tensors]() -> ::mediapipe::Status {
// Explicitly copy input.
for (int i = 0; i < input_tensors.size(); ++i) {
glBindBuffer(GL_COPY_READ_BUFFER,
input_tensors[i].GetOpenGlBufferReadView().name());
glBindBuffer(GL_COPY_WRITE_BUFFER,
gpu_buffers_in_[i]->GetOpenGlBufferWriteView().name());
glCopyBufferSubData(GL_COPY_READ_BUFFER, GL_COPY_WRITE_BUFFER, 0, 0,
input_tensors[i].bytes());
}
return absl::OkStatus();
}));
// Run inference.
RET_CHECK_EQ(interpreter_->Invoke(), kTfLiteOk);
MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext(
[this, &output_tensors]() -> ::mediapipe::Status {
output_tensors->reserve(output_shapes_.size());
for (int i = 0; i < output_shapes_.size(); ++i) {
const auto& t = gpu_buffers_out_[i];
output_tensors->emplace_back(Tensor::ElementType::kFloat32,
gpu_buffers_out_[i]->shape());
auto read_view = t->GetOpenGlBufferReadView();
glBindBuffer(GL_COPY_READ_BUFFER, read_view.name());
auto write_view = output_tensors->back().GetOpenGlBufferWriteView();
glBindBuffer(GL_COPY_WRITE_BUFFER, write_view.name());
glCopyBufferSubData(GL_COPY_READ_BUFFER, GL_COPY_WRITE_BUFFER, 0, 0,
t->bytes());
}
return absl::OkStatus();
}));
kOutTensors(cc).Send(std::move(output_tensors));
return absl::OkStatus();
}
absl::Status InferenceCalculatorGlImpl::Close(CalculatorContext* cc) {
return gpu_helper_.RunInGlContext([this]() -> absl::Status {
InferenceCalculatorGlImpl::GpuInferenceRunner::~GpuInferenceRunner() {
gpu_helper_.RunInGlContext([this]() {
gpu_buffers_in_.clear();
gpu_buffers_out_.clear();
// Delegate must outlive the interpreter, hence the order is important.
interpreter_ = nullptr;
delegate_ = nullptr;
return absl::OkStatus();
});
}
absl::Status InferenceCalculatorGlImpl::LoadModel(CalculatorContext* cc) {
absl::Status InferenceCalculatorGlImpl::GpuInferenceRunner::Init(
CalculatorContext* cc,
const mediapipe::InferenceCalculatorOptions::Delegate& delegate_options) {
MP_RETURN_IF_ERROR(LoadModel(cc));
MP_RETURN_IF_ERROR(gpu_helper_.Open(cc));
return gpu_helper_.RunInGlContext(
[this, &cc, &delegate_options]() -> absl::Status {
return LoadDelegateAndAllocateTensors(cc, delegate_options);
});
}
absl::Status InferenceCalculatorGlImpl::GpuInferenceRunner::LoadModel(
CalculatorContext* cc) {
ASSIGN_OR_RETURN(model_packet_, GetModelAsPacket(cc));
const auto& model = *model_packet_.Get();
if (kSideInOpResolver(cc).IsConnected()) {
@@ -160,9 +116,11 @@ absl::Status InferenceCalculatorGlImpl::LoadModel(CalculatorContext* cc) {
return absl::OkStatus();
}
absl::Status InferenceCalculatorGlImpl::LoadDelegateAndAllocateTensors(
CalculatorContext* cc) {
MP_RETURN_IF_ERROR(LoadDelegate(cc));
absl::Status
InferenceCalculatorGlImpl::GpuInferenceRunner::LoadDelegateAndAllocateTensors(
CalculatorContext* cc,
const mediapipe::InferenceCalculatorOptions::Delegate& delegate_options) {
MP_RETURN_IF_ERROR(LoadDelegate(cc, delegate_options));
// AllocateTensors() can be called only after ModifyGraphWithDelegate.
RET_CHECK_EQ(interpreter_->AllocateTensors(), kTfLiteOk);
@@ -173,11 +131,16 @@ absl::Status InferenceCalculatorGlImpl::LoadDelegateAndAllocateTensors(
return absl::OkStatus();
}
absl::Status InferenceCalculatorGlImpl::LoadDelegate(CalculatorContext* cc) {
absl::Status InferenceCalculatorGlImpl::GpuInferenceRunner::LoadDelegate(
CalculatorContext* cc,
const mediapipe::InferenceCalculatorOptions::Delegate& delegate_options) {
// Configure and create the delegate.
TfLiteGpuDelegateOptions options = TfLiteGpuDelegateOptionsDefault();
options.compile_options.precision_loss_allowed =
allow_precision_loss_ ? 1 : 0;
(delegate_options.has_gpu() &&
delegate_options.gpu().allow_precision_loss())
? 1
: 0;
options.compile_options.preferred_gl_object_type =
TFLITE_GL_OBJECT_TYPE_FASTEST;
options.compile_options.dynamic_batch_enabled = 0;
@@ -202,9 +165,9 @@ absl::Status InferenceCalculatorGlImpl::LoadDelegate(CalculatorContext* cc) {
interpreter_->SetAllowBufferHandleOutput(true);
// Get output image sizes.
const auto& output_indices = interpreter_->outputs();
output_shapes_.resize(output_indices.size());
output_size_ = output_indices.size();
// Create and bind output buffers.
for (int i = 0; i < output_shapes_.size(); ++i) {
for (int i = 0; i < output_size_; ++i) {
const TfLiteTensor* tensor = interpreter_->tensor(output_indices[i]);
gpu_buffers_out_.emplace_back(absl::make_unique<Tensor>(
Tensor::ElementType::kFloat32,
@@ -224,5 +187,89 @@ absl::Status InferenceCalculatorGlImpl::LoadDelegate(CalculatorContext* cc) {
return absl::OkStatus();
}
absl::Status InferenceCalculatorGlImpl::GpuInferenceRunner::Process(
CalculatorContext* cc, const std::vector<Tensor>& input_tensors,
std::vector<Tensor>& output_tensors) {
return gpu_helper_.RunInGlContext(
[this, &input_tensors, &output_tensors]() -> absl::Status {
// Explicitly copy input.
for (int i = 0; i < input_tensors.size(); ++i) {
glBindBuffer(GL_COPY_READ_BUFFER,
input_tensors[i].GetOpenGlBufferReadView().name());
glBindBuffer(GL_COPY_WRITE_BUFFER,
gpu_buffers_in_[i]->GetOpenGlBufferWriteView().name());
glCopyBufferSubData(GL_COPY_READ_BUFFER, GL_COPY_WRITE_BUFFER, 0, 0,
input_tensors[i].bytes());
}
// Run inference.
RET_CHECK_EQ(interpreter_->Invoke(), kTfLiteOk);
output_tensors.reserve(output_size_);
for (int i = 0; i < output_size_; ++i) {
const auto& t = gpu_buffers_out_[i];
output_tensors.emplace_back(Tensor::ElementType::kFloat32,
gpu_buffers_out_[i]->shape());
auto read_view = t->GetOpenGlBufferReadView();
glBindBuffer(GL_COPY_READ_BUFFER, read_view.name());
auto write_view = output_tensors.back().GetOpenGlBufferWriteView();
glBindBuffer(GL_COPY_WRITE_BUFFER, write_view.name());
glCopyBufferSubData(GL_COPY_READ_BUFFER, GL_COPY_WRITE_BUFFER, 0, 0,
t->bytes());
}
return absl::OkStatus();
});
}
absl::Status InferenceCalculatorGlImpl::UpdateContract(CalculatorContract* cc) {
const auto& options = cc->Options<mediapipe::InferenceCalculatorOptions>();
RET_CHECK(!options.model_path().empty() ^ kSideInModel(cc).IsConnected())
<< "Either model as side packet or model path in options is required.";
return mediapipe::GlCalculatorHelper::UpdateContract(cc);
}
absl::Status InferenceCalculatorGlImpl::Open(CalculatorContext* cc) {
const auto& options = cc->Options<mediapipe::InferenceCalculatorOptions>();
mediapipe::InferenceCalculatorOptions::Delegate delegate = options.delegate();
if (!kDelegate(cc).IsEmpty()) {
const mediapipe::InferenceCalculatorOptions::Delegate&
input_side_packet_delegate = kDelegate(cc).Get();
RET_CHECK(
(input_side_packet_delegate.has_gpu() &&
!input_side_packet_delegate.gpu().use_advanced_gpu_api()) ||
input_side_packet_delegate.delegate_case() ==
mediapipe::InferenceCalculatorOptions::Delegate::DELEGATE_NOT_SET)
<< "inference_calculator_gl only supports delegate input side packet "
<< "for Gpu (non advanced)";
delegate.MergeFrom(input_side_packet_delegate);
}
gpu_inference_runner_ = std::make_unique<GpuInferenceRunner>();
return gpu_inference_runner_->Init(cc, delegate);
}
absl::Status InferenceCalculatorGlImpl::Process(CalculatorContext* cc) {
if (kInTensors(cc).IsEmpty()) {
return absl::OkStatus();
}
const auto& input_tensors = *kInTensors(cc);
RET_CHECK(!input_tensors.empty());
auto output_tensors = absl::make_unique<std::vector<Tensor>>();
MP_RETURN_IF_ERROR(
gpu_inference_runner_->Process(cc, input_tensors, *output_tensors));
kOutTensors(cc).Send(std::move(output_tensors));
return absl::OkStatus();
}
absl::Status InferenceCalculatorGlImpl::Close(CalculatorContext* cc) {
gpu_inference_runner_ = nullptr;
return absl::OkStatus();
}
} // namespace api2
} // namespace mediapipe
mediapipe/calculators/tensor/inference_calculator_gl_advanced.cc
+218 -135
View File
@@ -15,10 +15,12 @@
#include <cstring>
#include <memory>
#include <string>
#include <utility>
#include <vector>
#include "absl/memory/memory.h"
#include "absl/status/status.h"
#include "absl/status/statusor.h"
#include "mediapipe/calculators/tensor/inference_calculator.h"
#include "mediapipe/gpu/gl_calculator_helper.h"
#include "mediapipe/util/tflite/tflite_gpu_runner.h"
@@ -28,7 +30,7 @@
#include "mediapipe/util/android/file/base/file.h"
#include "mediapipe/util/android/file/base/filesystem.h"
#include "mediapipe/util/android/file/base/helpers.h"
#endif // ANDROID
#endif // MEDIAPIPE_ANDROID
namespace mediapipe {
namespace api2 {
@@ -56,85 +58,71 @@ class InferenceCalculatorGlAdvancedImpl
absl::Status Close(CalculatorContext* cc) override;
private:
absl::Status ReadGpuCaches();
absl::Status SaveGpuCaches();
absl::Status InitTFLiteGPURunner(CalculatorContext* cc);
// Helper class that saves binary data to disk, or read from disk.
class OnDiskCacheHelper {
public:
absl::Status Init(
const mediapipe::InferenceCalculatorOptions& options,
const mediapipe::InferenceCalculatorOptions::Delegate::Gpu&
gpu_delegate_options);
absl::Status ReadGpuCaches(tflite::gpu::TFLiteGPURunner* gpu_runner) const;
absl::Status SaveGpuCaches(tflite::gpu::TFLiteGPURunner* gpu_runner) const;
// TfLite requires us to keep the model alive as long as the interpreter is.
Packet<TfLiteModelPtr> model_packet_;
private:
bool use_kernel_caching_ = false;
std::string cached_kernel_filename_;
bool use_serialized_model_ = false;
std::string serialized_model_path_;
};
mediapipe::GlCalculatorHelper gpu_helper_;
std::unique_ptr<tflite::gpu::TFLiteGPURunner> tflite_gpu_runner_;
bool allow_precision_loss_ = false;
mediapipe::InferenceCalculatorOptions::Delegate::Gpu::Api
tflite_gpu_runner_api_;
mediapipe::InferenceCalculatorOptions::Delegate::Gpu::InferenceUsage
tflite_gpu_runner_usage_;
// Helper class that wraps everything related to GPU inference acceleration.
class GpuInferenceRunner {
public:
absl::Status Init(
CalculatorContext* cc,
const mediapipe::InferenceCalculatorOptions::Delegate& delegate);
std::vector<Tensor::Shape> output_shapes_;
absl::StatusOr<std::vector<Tensor>> Process(
const std::vector<Tensor>& input_tensors);
bool use_kernel_caching_ = false;
std::string cached_kernel_filename_;
bool use_serialized_model_ = false;
std::string serialized_model_path_;
absl::Status Close();
private:
absl::Status InitTFLiteGPURunner(
CalculatorContext* cc,
const mediapipe::InferenceCalculatorOptions::Delegate& delegate);
// TfLite requires us to keep the model alive as long as the interpreter is.
Packet<TfLiteModelPtr> model_packet_;
mediapipe::GlCalculatorHelper gpu_helper_;
std::unique_ptr<tflite::gpu::TFLiteGPURunner> tflite_gpu_runner_;
std::vector<Tensor::Shape> output_shapes_;
OnDiskCacheHelper on_disk_cache_helper_;
};
std::unique_ptr<GpuInferenceRunner> gpu_inference_runner_;
};
absl::Status InferenceCalculatorGlAdvancedImpl::UpdateContract(
CalculatorContract* cc) {
const auto& options = cc->Options<::mediapipe::InferenceCalculatorOptions>();
RET_CHECK(!options.model_path().empty() ^ kSideInModel(cc).IsConnected())
<< "Either model as side packet or model path in options is required.";
MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc));
return absl::OkStatus();
}
absl::Status InferenceCalculatorGlAdvancedImpl::Open(CalculatorContext* cc) {
const auto& options = cc->Options<::mediapipe::InferenceCalculatorOptions>();
mediapipe::InferenceCalculatorOptions::Delegate delegate = options.delegate();
if (!kDelegate(cc).IsEmpty()) {
mediapipe::InferenceCalculatorOptions::Delegate input_side_packet_delegate =
kDelegate(cc).Get();
CHECK(input_side_packet_delegate.has_gpu() ||
input_side_packet_delegate.delegate_case() ==
mediapipe::InferenceCalculatorOptions::Delegate::DELEGATE_NOT_SET)
<< "inference_calculator_gl_advanced only supports delegate input side "
"packet for Gpu";
delegate.MergeFrom(input_side_packet_delegate);
}
allow_precision_loss_ = delegate.gpu().allow_precision_loss();
tflite_gpu_runner_api_ = delegate.gpu().api();
tflite_gpu_runner_usage_ = delegate.gpu().usage();
use_kernel_caching_ = delegate.gpu().has_cached_kernel_path();
use_serialized_model_ = delegate.gpu().has_serialized_model_dir() &&
delegate.gpu().has_model_token();
if (use_kernel_caching_) {
#ifdef MEDIAPIPE_ANDROID
cached_kernel_filename_ = delegate.gpu().cached_kernel_path() +
mediapipe::File::Basename(options.model_path()) +
".ker";
#endif // MEDIAPIPE_ANDROID
}
if (use_serialized_model_) {
#ifdef MEDIAPIPE_ANDROID
serialized_model_path_ = mediapipe::file::JoinPath(
delegate.gpu().serialized_model_dir(), delegate.gpu().model_token());
#endif // MEDIAPIPE_ANDROID
}
absl::Status InferenceCalculatorGlAdvancedImpl::GpuInferenceRunner::Init(
CalculatorContext* cc,
const mediapipe::InferenceCalculatorOptions::Delegate& delegate) {
MP_RETURN_IF_ERROR(gpu_helper_.Open(cc));
return gpu_helper_.RunInGlContext(
[this, &cc]() -> absl::Status { return InitTFLiteGPURunner(cc); });
const auto& options = cc->Options<mediapipe::InferenceCalculatorOptions>();
MP_RETURN_IF_ERROR(on_disk_cache_helper_.Init(options, delegate.gpu()));
return gpu_helper_.RunInGlContext([this, &cc, &delegate]() -> absl::Status {
return InitTFLiteGPURunner(cc, delegate);
});
}
absl::Status InferenceCalculatorGlAdvancedImpl::Process(CalculatorContext* cc) {
if (kInTensors(cc).IsEmpty()) {
return absl::OkStatus();
}
const auto& input_tensors = *kInTensors(cc);
RET_CHECK(!input_tensors.empty());
auto output_tensors = absl::make_unique<std::vector<Tensor>>();
absl::StatusOr<std::vector<Tensor>>
InferenceCalculatorGlAdvancedImpl::GpuInferenceRunner::Process(
const std::vector<Tensor>& input_tensors) {
std::vector<Tensor> output_tensors;
MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext(
[this, &input_tensors, &output_tensors]() -> absl::Status {
@@ -142,90 +130,46 @@ absl::Status InferenceCalculatorGlAdvancedImpl::Process(CalculatorContext* cc) {
MP_RETURN_IF_ERROR(tflite_gpu_runner_->BindSSBOToInputTensor(
input_tensors[i].GetOpenGlBufferReadView().name(), i));
}
output_tensors->reserve(output_shapes_.size());
output_tensors.reserve(output_shapes_.size());
for (int i = 0; i < output_shapes_.size(); ++i) {
output_tensors->emplace_back(Tensor::ElementType::kFloat32,
output_shapes_[i]);
output_tensors.emplace_back(Tensor::ElementType::kFloat32,
output_shapes_[i]);
MP_RETURN_IF_ERROR(tflite_gpu_runner_->BindSSBOToOutputTensor(
output_tensors->back().GetOpenGlBufferWriteView().name(), i));
output_tensors.back().GetOpenGlBufferWriteView().name(), i));
}
return absl::OkStatus();
// Run inference.
return tflite_gpu_runner_->Invoke();
}));
// Run inference.
MP_RETURN_IF_ERROR(tflite_gpu_runner_->Invoke());
kOutTensors(cc).Send(std::move(output_tensors));
return absl::OkStatus();
return output_tensors;
}
absl::Status InferenceCalculatorGlAdvancedImpl::SaveGpuCaches() {
#ifdef MEDIAPIPE_ANDROID
if (use_kernel_caching_) {
// Save kernel file.
auto kernel_cache = absl::make_unique<std::vector<uint8_t>>(
tflite_gpu_runner_->GetSerializedBinaryCache());
std::string cache_str(kernel_cache->begin(), kernel_cache->end());
MP_RETURN_IF_ERROR(
mediapipe::file::SetContents(cached_kernel_filename_, cache_str));
}
if (use_serialized_model_) {
// Save serialized model file.
ASSIGN_OR_RETURN(std::vector<uint8_t> serialized_model_vec,
tflite_gpu_runner_->GetSerializedModel());
absl::string_view serialized_model(
reinterpret_cast<char*>(serialized_model_vec.data()),
serialized_model_vec.size());
MP_RETURN_IF_ERROR(
mediapipe::file::SetContents(serialized_model_path_, serialized_model));
}
#endif // MEDIAPIPE_ANDROID
return absl::OkStatus();
}
absl::Status InferenceCalculatorGlAdvancedImpl::Close(CalculatorContext* cc) {
MP_RETURN_IF_ERROR(SaveGpuCaches());
absl::Status InferenceCalculatorGlAdvancedImpl::GpuInferenceRunner::Close() {
MP_RETURN_IF_ERROR(
on_disk_cache_helper_.SaveGpuCaches(tflite_gpu_runner_.get()));
return gpu_helper_.RunInGlContext([this]() -> absl::Status {
tflite_gpu_runner_.reset();
return absl::OkStatus();
});
}
absl::Status InferenceCalculatorGlAdvancedImpl::ReadGpuCaches() {
#ifdef MEDIAPIPE_ANDROID
if (use_kernel_caching_ && File::Exists(cached_kernel_filename_)) {
// Load pre-compiled kernel file.
std::string cache_str;
MP_RETURN_IF_ERROR(
mediapipe::file::GetContents(cached_kernel_filename_, &cache_str));
std::vector<uint8_t> cache_vec(cache_str.begin(), cache_str.end());
tflite_gpu_runner_->SetSerializedBinaryCache(std::move(cache_vec));
}
if (use_serialized_model_ && File::Exists(serialized_model_path_)) {
// Load serialized model file.
std::string serialized_model_str;
MP_RETURN_IF_ERROR(
file::GetContents(serialized_model_path_, &serialized_model_str));
std::vector<uint8_t> serialized_model_vec(serialized_model_str.begin(),
serialized_model_str.end());
tflite_gpu_runner_->SetSerializedModel(std::move(serialized_model_vec));
}
#endif // MEDIAPIPE_ANDROID
return absl::OkStatus();
}
absl::Status InferenceCalculatorGlAdvancedImpl::InitTFLiteGPURunner(
CalculatorContext* cc) {
absl::Status
InferenceCalculatorGlAdvancedImpl::GpuInferenceRunner::InitTFLiteGPURunner(
CalculatorContext* cc,
const mediapipe::InferenceCalculatorOptions::Delegate& delegate) {
ASSIGN_OR_RETURN(model_packet_, GetModelAsPacket(cc));
const auto& model = *model_packet_.Get();
bool allow_precision_loss = delegate.gpu().allow_precision_loss();
// Create runner
tflite::gpu::InferenceOptions options;
options.priority1 = allow_precision_loss_
options.priority1 = allow_precision_loss
? tflite::gpu::InferencePriority::MIN_LATENCY
: tflite::gpu::InferencePriority::MAX_PRECISION;
options.priority2 = tflite::gpu::InferencePriority::AUTO;
options.priority3 = tflite::gpu::InferencePriority::AUTO;
switch (tflite_gpu_runner_usage_) {
switch (delegate.gpu().usage()) {
case mediapipe::InferenceCalculatorOptions::Delegate::Gpu::
FAST_SINGLE_ANSWER: {
options.usage = tflite::gpu::InferenceUsage::FAST_SINGLE_ANSWER;
@@ -241,7 +185,7 @@ absl::Status InferenceCalculatorGlAdvancedImpl::InitTFLiteGPURunner(
}
}
tflite_gpu_runner_ = std::make_unique<tflite::gpu::TFLiteGPURunner>(options);
switch (tflite_gpu_runner_api_) {
switch (delegate.gpu().api()) {
case mediapipe::InferenceCalculatorOptions::Delegate::Gpu::ANY: {
// Do not need to force any specific API.
break;
@@ -277,9 +221,148 @@ absl::Status InferenceCalculatorGlAdvancedImpl::InitTFLiteGPURunner(
tflite_gpu_runner_->GetOutputShapes()[i].c};
}
MP_RETURN_IF_ERROR(ReadGpuCaches());
MP_RETURN_IF_ERROR(
on_disk_cache_helper_.ReadGpuCaches(tflite_gpu_runner_.get()));
return tflite_gpu_runner_->Build();
}
#if defined(MEDIAPIPE_ANDROID)
absl::Status InferenceCalculatorGlAdvancedImpl::OnDiskCacheHelper::Init(
const mediapipe::InferenceCalculatorOptions& options,
const mediapipe::InferenceCalculatorOptions::Delegate::Gpu&
gpu_delegate_options) {
use_kernel_caching_ = gpu_delegate_options.has_cached_kernel_path();
use_serialized_model_ = gpu_delegate_options.has_serialized_model_dir() &&
gpu_delegate_options.has_model_token();
if (use_kernel_caching_) {
cached_kernel_filename_ = gpu_delegate_options.cached_kernel_path() +
mediapipe::File::Basename(options.model_path()) +
".ker";
}
if (use_serialized_model_) {
serialized_model_path_ =
mediapipe::file::JoinPath(gpu_delegate_options.serialized_model_dir(),
gpu_delegate_options.model_token());
}
return absl::OkStatus();
}
absl::Status
InferenceCalculatorGlAdvancedImpl::OnDiskCacheHelper::SaveGpuCaches(
tflite::gpu::TFLiteGPURunner* gpu_runner) const {
if (use_kernel_caching_) {
// Save kernel file.
auto kernel_cache = absl::make_unique<std::vector<uint8_t>>(
gpu_runner->GetSerializedBinaryCache());
std::string cache_str(kernel_cache->begin(), kernel_cache->end());
MP_RETURN_IF_ERROR(
mediapipe::file::SetContents(cached_kernel_filename_, cache_str));
}
if (use_serialized_model_) {
// Save serialized model file.
ASSIGN_OR_RETURN(std::vector<uint8_t> serialized_model_vec,
gpu_runner->GetSerializedModel());
absl::string_view serialized_model(
reinterpret_cast<char*>(serialized_model_vec.data()),
serialized_model_vec.size());
MP_RETURN_IF_ERROR(
mediapipe::file::SetContents(serialized_model_path_, serialized_model));
}
return absl::OkStatus();
}
absl::Status
InferenceCalculatorGlAdvancedImpl::OnDiskCacheHelper::ReadGpuCaches(
tflite::gpu::TFLiteGPURunner* gpu_runner) const {
if (use_kernel_caching_ && File::Exists(cached_kernel_filename_)) {
// Load pre-compiled kernel file.
std::string cache_str;
MP_RETURN_IF_ERROR(
mediapipe::file::GetContents(cached_kernel_filename_, &cache_str));
std::vector<uint8_t> cache_vec(cache_str.begin(), cache_str.end());
gpu_runner->SetSerializedBinaryCache(std::move(cache_vec));
}
if (use_serialized_model_ && File::Exists(serialized_model_path_)) {
// Load serialized model file.
std::string serialized_model_str;
MP_RETURN_IF_ERROR(
file::GetContents(serialized_model_path_, &serialized_model_str));
std::vector<uint8_t> serialized_model_vec(serialized_model_str.begin(),
serialized_model_str.end());
gpu_runner->SetSerializedModel(std::move(serialized_model_vec));
}
return absl::OkStatus();
}
#else
absl::Status InferenceCalculatorGlAdvancedImpl::OnDiskCacheHelper::Init(
const mediapipe::InferenceCalculatorOptions& options,
const mediapipe::InferenceCalculatorOptions::Delegate::Gpu&
gpu_delegate_options) {
return absl::OkStatus();
}
absl::Status
InferenceCalculatorGlAdvancedImpl::OnDiskCacheHelper::ReadGpuCaches(
tflite::gpu::TFLiteGPURunner* gpu_runner) const {
return absl::OkStatus();
}
absl::Status
InferenceCalculatorGlAdvancedImpl::OnDiskCacheHelper::SaveGpuCaches(
tflite::gpu::TFLiteGPURunner* gpu_runner) const {
return absl::OkStatus();
}
#endif // MEDIAPIPE_ANDROID
absl::Status InferenceCalculatorGlAdvancedImpl::UpdateContract(
CalculatorContract* cc) {
const auto& options = cc->Options<mediapipe::InferenceCalculatorOptions>();
RET_CHECK(!options.model_path().empty() ^ kSideInModel(cc).IsConnected())
<< "Either model as side packet or model path in options is required.";
MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc));
return absl::OkStatus();
}
absl::Status InferenceCalculatorGlAdvancedImpl::Open(CalculatorContext* cc) {
const auto& options = cc->Options<mediapipe::InferenceCalculatorOptions>();
mediapipe::InferenceCalculatorOptions::Delegate delegate = options.delegate();
if (!kDelegate(cc).IsEmpty()) {
const mediapipe::InferenceCalculatorOptions::Delegate&
input_side_packet_delegate = kDelegate(cc).Get();
RET_CHECK(
input_side_packet_delegate.has_gpu() ||
input_side_packet_delegate.delegate_case() ==
mediapipe::InferenceCalculatorOptions::Delegate::DELEGATE_NOT_SET)
<< "inference_calculator_gl_advanced only supports gpu delegate "
"configuration through side packet.";
delegate.MergeFrom(input_side_packet_delegate);
}
gpu_inference_runner_ = std::make_unique<GpuInferenceRunner>();
return gpu_inference_runner_->Init(cc, delegate);
}
absl::Status InferenceCalculatorGlAdvancedImpl::Process(CalculatorContext* cc) {
if (kInTensors(cc).IsEmpty()) {
return absl::OkStatus();
}
const auto& input_tensors = *kInTensors(cc);
RET_CHECK(!input_tensors.empty());
auto output_tensors = absl::make_unique<std::vector<Tensor>>();
ASSIGN_OR_RETURN(*output_tensors,
gpu_inference_runner_->Process(input_tensors));
kOutTensors(cc).Send(std::move(output_tensors));
return absl::OkStatus();
}
absl::Status InferenceCalculatorGlAdvancedImpl::Close(CalculatorContext* cc) {
return gpu_inference_runner_->Close();
}
} // namespace api2
} // namespace mediapipe
mediapipe/calculators/tensor/inference_calculator_metal.cc
+3 -1
View File
@@ -116,7 +116,9 @@ class InferenceCalculatorMetalImpl
absl::Status InferenceCalculatorMetalImpl::UpdateContract(
CalculatorContract* cc) {
const auto& options = cc->Options<::mediapipe::InferenceCalculatorOptions>();
RET_CHECK(!kDelegate(cc).IsConnected())
<< "Delegate configuration through side packet is not supported.";
const auto& options = cc->Options<mediapipe::InferenceCalculatorOptions>();
RET_CHECK(!options.model_path().empty() ^ kSideInModel(cc).IsConnected())
<< "Either model as side packet or model path in options is required.";
mediapipe/calculators/tensor/inference_calculator_test.cc
+17 -3
View File
@@ -16,13 +16,17 @@
#include <string>
#include <vector>
#include "absl/log/check.h"
#include "absl/strings/str_cat.h"
#include "absl/strings/str_replace.h"
#include "absl/strings/string_view.h"
#include "mediapipe/calculators/tensor/inference_calculator.pb.h"
#include "mediapipe/calculators/tensor/inference_calculator_test_base.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_runner.h"
#include "mediapipe/framework/deps/file_path.h"
#include "mediapipe/framework/formats/tensor.h"
#include "mediapipe/framework/port/benchmark.h"
#include "mediapipe/framework/port/gmock.h"
#include "mediapipe/framework/port/gtest.h"
#include "mediapipe/framework/port/integral_types.h"
@@ -118,9 +122,11 @@ void RunGraphThenClose(CalculatorGraph& graph, std::vector<Tensor> input_vec) {
MP_ASSERT_OK(graph.StartRun({}));
// Push the tensor into the graph.
MP_ASSERT_OK(graph.AddPacketToInputStream(
"tensor_in",
MakePacket<std::vector<Tensor>>(std::move(input_vec)).At(Timestamp(0))));
if (!input_vec.empty()) {
MP_ASSERT_OK(graph.AddPacketToInputStream(
"tensor_in", MakePacket<std::vector<Tensor>>(std::move(input_vec))
.At(Timestamp(0))));
}
// Wait until the calculator done processing.
MP_ASSERT_OK(graph.WaitUntilIdle());
@@ -174,5 +180,13 @@ TEST(InferenceCalculatorTest, ModelAsInputSidePacketSmokeTest) {
DoSmokeTest(kGraphWithModelAsInputSidePacket);
}
void BM_InitializeCalculator(benchmark::State& state) {
mediapipe::InferenceCalculatorOptions::Delegate delegate;
delegate.mutable_tflite();
RunBenchmarkCalculatorInitialization(state, delegate);
}
BENCHMARK(BM_InitializeCalculator);
} // namespace
} // namespace mediapipe
mediapipe/calculators/tensor/landmarks_to_tensor_calculator.cc
+85 -26
View File
@@ -15,6 +15,8 @@
#include "mediapipe/calculators/tensor/landmarks_to_tensor_calculator.h"
#include <memory>
#include <optional>
#include <type_traits>
#include "mediapipe/calculators/tensor/landmarks_to_tensor_calculator.pb.h"
#include "mediapipe/framework/api2/node.h"
@@ -28,8 +30,25 @@ namespace api2 {
namespace {
// Returns the scale attribute should be multiplied by.
float GetAttributeScale(
const LandmarksToTensorCalculatorOptions::Attribute& attribute,
const std::pair<int, int>& image_size) {
switch (attribute) {
case LandmarksToTensorCalculatorOptions::X:
case LandmarksToTensorCalculatorOptions::Z:
return image_size.first;
case LandmarksToTensorCalculatorOptions::Y:
return image_size.second;
case LandmarksToTensorCalculatorOptions::VISIBILITY:
case LandmarksToTensorCalculatorOptions::PRESENCE:
return 1.0f;
}
}
template <typename LandmarkType>
float GetAttribute(
const Landmark& landmark,
const LandmarkType& landmark,
const LandmarksToTensorCalculatorOptions::Attribute& attribute) {
switch (attribute) {
case LandmarksToTensorCalculatorOptions::X:
@@ -45,6 +64,33 @@ float GetAttribute(
}
}
template <typename LandmarksT>
Tensor ConvertLandmarksToTensor(
const LandmarksT& landmarks, const std::vector<float>& attribute_scales,
const LandmarksToTensorCalculatorOptions& options) {
// Determine tensor shape.
const int n_landmarks = landmarks.landmark_size();
const int n_attributes = options.attributes_size();
auto tensor_shape = options.flatten()
? Tensor::Shape{1, n_landmarks * n_attributes}
: Tensor::Shape{1, n_landmarks, n_attributes};
// Create empty tesnor.
Tensor tensor(Tensor::ElementType::kFloat32, tensor_shape);
auto* buffer = tensor.GetCpuWriteView().buffer<float>();
// Fill tensor with landmark attributes.
for (int i = 0; i < n_landmarks; ++i) {
for (int j = 0; j < n_attributes; ++j) {
float value = GetAttribute(landmarks.landmark(i), options.attributes(j));
float scale = attribute_scales[j];
buffer[i * n_attributes + j] = value * scale;
}
}
return tensor;
}
} // namespace
class LandmarksToTensorCalculatorImpl
@@ -54,39 +100,52 @@ class LandmarksToTensorCalculatorImpl
options_ = cc->Options<LandmarksToTensorCalculatorOptions>();
RET_CHECK(options_.attributes_size() > 0)
<< "At least one attribute must be specified";
RET_CHECK(kInLandmarkList(cc).IsConnected() ^
kInNormLandmarkList(cc).IsConnected())
<< "Exactly one landmarks input should be provided";
RET_CHECK_EQ(kInNormLandmarkList(cc).IsConnected(),
kImageSize(cc).IsConnected())
<< "Image size should be provided only for normalized landmarks";
return absl::OkStatus();
}
absl::Status Process(CalculatorContext* cc) override {
if (kInLandmarkList(cc).IsEmpty()) {
return absl::OkStatus();
}
// Get input landmarks.
const auto& in_landmarks = *kInLandmarkList(cc);
// Determine tensor shape.
const int n_landmarks = in_landmarks.landmark_size();
const int n_attributes = options_.attributes_size();
auto tensor_shape = options_.flatten()
? Tensor::Shape{1, n_landmarks * n_attributes}
: Tensor::Shape{1, n_landmarks, n_attributes};
// Create empty tesnor.
Tensor tensor(Tensor::ElementType::kFloat32, tensor_shape);
auto* buffer = tensor.GetCpuWriteView().buffer<float>();
// Fill tensor with landmark attributes.
for (int i = 0; i < n_landmarks; ++i) {
for (int j = 0; j < n_attributes; ++j) {
buffer[i * n_attributes + j] =
GetAttribute(in_landmarks.landmark(i), options_.attributes(j));
// Get attribute scales depending on whether landmarks are normalized or
// not.
std::vector<float> attribute_scales;
if (kInLandmarkList(cc).IsConnected()) {
for (int j = 0; j < options_.attributes_size(); ++j) {
attribute_scales.push_back(1.0f);
}
} else {
RET_CHECK(!kImageSize(cc).IsEmpty());
auto image_size = kImageSize(cc).Get();
for (int j = 0; j < options_.attributes_size(); ++j) {
attribute_scales.push_back(
GetAttributeScale(options_.attributes(j), image_size));
}
}
// Return vector with a single tensor.
// Convert landmarks to tensor.
auto result = std::vector<Tensor>();
result.push_back(std::move(tensor));
if (kInLandmarkList(cc).IsConnected()) {
if (kInLandmarkList(cc).IsEmpty()) {
return absl::OkStatus();
}
Tensor tensor = ConvertLandmarksToTensor(kInLandmarkList(cc).Get(),
attribute_scales, options_);
result.push_back(std::move(tensor));
} else {
if (kInNormLandmarkList(cc).IsEmpty()) {
return absl::OkStatus();
}
Tensor tensor = ConvertLandmarksToTensor(kInNormLandmarkList(cc).Get(),
attribute_scales, options_);
result.push_back(std::move(tensor));
}
kOutTensors(cc).Send(std::move(result));
return absl::OkStatus();
mediapipe/calculators/tensor/landmarks_to_tensor_calculator.h
+12 -3
View File
@@ -28,8 +28,12 @@ namespace api2 {
// A calculator for converting landmars into a Tensor.
//
// Input:
// LANDMARKS - LandmarkList
// LANDMARKS (optional) - LandmarkList
// Landmarks to be converted into a Tensor.
// NORM_LANDMARKS (optional) - NormalizedLandmarkList.
// Normalized landmarks to be converted into a Tensor.
// IMAGE_SIZE (optional) - std::pair<int, int>
// Image size to scale NORM_LANDMARKS.
//
// Output:
// TENSORS - std::vector<Tensor>
@@ -49,10 +53,15 @@ namespace api2 {
// }
class LandmarksToTensorCalculator : public NodeIntf {
public:
static constexpr Input<LandmarkList>::Optional kInLandmarkList{"LANDMARKS"};
static constexpr Input<mediapipe::LandmarkList>::Optional kInLandmarkList{
"LANDMARKS"};
static constexpr Input<mediapipe::NormalizedLandmarkList>::Optional
kInNormLandmarkList{"NORM_LANDMARKS"};
static constexpr Input<std::pair<int, int>>::Optional kImageSize{
"IMAGE_SIZE"};
static constexpr Output<std::vector<Tensor>> kOutTensors{"TENSORS"};
MEDIAPIPE_NODE_INTERFACE(LandmarksToTensorCalculator, kInLandmarkList,
kOutTensors);
kInNormLandmarkList, kImageSize, kOutTensors);
};
} // namespace api2
mediapipe/calculators/tensor/landmarks_to_tensor_calculator_test.cc
+43
View File
@@ -40,6 +40,20 @@ void RunLandmarks(mediapipe::CalculatorRunner* runner,
MP_ASSERT_OK(runner->Run());
}
void RunNormLandmarks(mediapipe::CalculatorRunner* runner,
const NormalizedLandmarkList& landmarks,
const std::pair<int, int> image_size) {
runner->MutableInputs()
->Tag("NORM_LANDMARKS")
.packets.push_back(
MakePacket<NormalizedLandmarkList>(landmarks).At(Timestamp(0)));
runner->MutableInputs()
->Tag("IMAGE_SIZE")
.packets.push_back(
MakePacket<std::pair<int, int>>(image_size).At(Timestamp(0)));
MP_ASSERT_OK(runner->Run());
}
const Tensor& GetOutputTensor(mediapipe::CalculatorRunner* runner) {
const auto& output_packets = runner->Outputs().Tag("TENSORS").packets;
EXPECT_EQ(output_packets.size(), 1);
@@ -151,5 +165,34 @@ TEST(LandmarksToTensorCalculatorTest, XYZAttributes_Flatten) {
{1.0f, 2.0f, 3.0f, 6.0f, 7.0f, 8.0f});
}
TEST(LandmarksToTensorCalculatorTest, NormalizedLandmarks) {
mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"pb(
calculator: "LandmarksToTensorCalculator"
input_stream: "NORM_LANDMARKS:landmarks"
input_stream: "IMAGE_SIZE:image_size"
output_stream: "TENSORS:tensors"
options: {
[mediapipe.LandmarksToTensorCalculatorOptions.ext] {
attributes: [ X, Y, Z, VISIBILITY, PRESENCE ]
}
}
)pb"));
NormalizedLandmarkList landmarks;
auto* landmark1 = landmarks.add_landmark();
landmark1->set_x(0.1f);
landmark1->set_y(0.5f);
landmark1->set_z(1.0f);
landmark1->set_visibility(4.0f);
landmark1->set_presence(5.0f);
std::pair<int, int> image_size{200, 100};
RunNormLandmarks(&runner, landmarks, image_size);
const auto& tensor = GetOutputTensor(&runner);
ValidateTensor(tensor, /*expected_shape=*/{1, 1, 5}, /*expected_values=*/
{20.0f, 50.0f, 200.0f, 4.0f, 5.0f});
}
} // namespace
} // namespace mediapipe
mediapipe/calculators/tensor/tensors_dequantization_calculator.cc
+102
View File
@@ -0,0 +1,102 @@
// Copyright 2022 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <memory>
#include <vector>
#include "absl/status/status.h"
#include "absl/strings/str_cat.h"
#include "mediapipe/framework/api2/node.h"
#include "mediapipe/framework/api2/port.h"
#include "mediapipe/framework/calculator_context.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/tensor.h"
#include "mediapipe/framework/port/ret_check.h"
namespace mediapipe {
namespace api2 {
namespace {
template <typename T>
void Dequantize(const Tensor& input, Tensor* output) {
auto input_view = input.GetCpuReadView();
auto input_buffer = input_view.buffer<T>();
auto output_view = output->GetCpuWriteView();
auto output_buffer = output_view.buffer<float>();
for (int i = 0; i < input.shape().num_elements(); ++i) {
output_buffer[i] = input.quantization_parameters().scale *
(static_cast<int>(input_buffer[i]) -
input.quantization_parameters().zero_point);
}
}
} // namespace
// Performs dequantization using the quantization parameters from the input
// UInt8 or Int8 tensors. Each element of the input tensors is converted using:
//
// output = quantization_parameters.scale *
// (input - quantization_parameters.zero_point)
//
// Input:
// TENSORS - Vector of quantized Tensors of type kUint8 or kInt8.
// Output:
// TENSORS - Vector of dequantized Tensors of type kFloat32.
//
// Usage example:
// node {
// calculator: "TensorsDequantizationCalculator"
// input_stream: "TENSORS:quantized_tensors"
// output_stream: "TENSORS:dequantized_tensors"
// }
class TensorsDequantizationCalculator : public Node {
public:
static constexpr Input<std::vector<Tensor>> kInTensors{"TENSORS"};
static constexpr Output<std::vector<Tensor>> kOutTensors{"TENSORS"};
MEDIAPIPE_NODE_CONTRACT(kInTensors, kOutTensors);
absl::Status Process(CalculatorContext* cc) override;
};
absl::Status TensorsDequantizationCalculator::Process(CalculatorContext* cc) {
if (kInTensors(cc).IsEmpty()) {
return absl::OkStatus();
}
const auto& input_tensors = *kInTensors(cc);
RET_CHECK(!input_tensors.empty());
auto output_tensors = std::make_unique<std::vector<Tensor>>();
output_tensors->reserve(input_tensors.size());
for (const auto& input_tensor : input_tensors) {
output_tensors->emplace_back(Tensor::ElementType::kFloat32,
input_tensor.shape());
switch (input_tensor.element_type()) {
case Tensor::ElementType::kUInt8:
Dequantize<uint8>(input_tensor, &output_tensors->back());
break;
case Tensor::ElementType::kInt8:
Dequantize<int8>(input_tensor, &output_tensors->back());
break;
default:
return absl::InvalidArgumentError(absl::StrCat(
"Unsupported input tensor type: ", input_tensor.element_type()));
}
}
kOutTensors(cc).Send(std::move(output_tensors));
return absl::OkStatus();
}
MEDIAPIPE_REGISTER_NODE(TensorsDequantizationCalculator);
} // namespace api2
} // namespace mediapipe
mediapipe/calculators/tensor/tensors_dequantization_calculator_test.cc
+128
View File
@@ -0,0 +1,128 @@
// Copyright 2022 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <cstdint>
#include <memory>
#include <vector>
#include "absl/status/status.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_runner.h"
#include "mediapipe/framework/formats/tensor.h"
#include "mediapipe/framework/port/gmock.h"
#include "mediapipe/framework/port/gtest.h"
#include "mediapipe/framework/port/parse_text_proto.h"
#include "mediapipe/framework/port/status_matchers.h"
namespace mediapipe {
namespace {
using ::mediapipe::ParseTextProtoOrDie;
using ::testing::HasSubstr;
using Node = ::mediapipe::CalculatorGraphConfig::Node;
constexpr char kCalculatorConfig[] = R"pb(
calculator: "TensorsDequantizationCalculator"
input_stream: "TENSORS:input"
output_stream: "TENSORS:output"
)pb";
// Compares the provided tensor contents with the expected values.
void ValidateResult(const Tensor& actual, const std::vector<float>& expected) {
EXPECT_EQ(actual.element_type(), Tensor::ElementType::kFloat32);
EXPECT_EQ(expected.size(), actual.shape().num_elements());
auto view = actual.GetCpuReadView();
auto buffer = view.buffer<float>();
for (int i = 0; i < expected.size(); ++i) {
EXPECT_FLOAT_EQ(expected[i], buffer[i]);
}
}
class TensorsDequantizationCalculatorTest : public ::testing::Test {
protected:
TensorsDequantizationCalculatorTest()
: runner_(ParseTextProtoOrDie<Node>(kCalculatorConfig)) {}
template <typename T>
void PushTensor(Tensor::ElementType type, std::vector<T> tensor,
std::optional<Tensor::QuantizationParameters>
quantization_params = std::nullopt) {
auto tensors = std::make_unique<std::vector<Tensor>>();
if (quantization_params.has_value()) {
tensors->emplace_back(type,
Tensor::Shape{static_cast<int>(tensor.size())},
quantization_params.value());
} else {
tensors->emplace_back(type,
Tensor::Shape{static_cast<int>(tensor.size())});
}
auto view = tensors->back().GetCpuWriteView();
auto buffer = view.buffer<T>();
std::copy(tensor.begin(), tensor.end(), buffer);
runner_.MutableInputs()->Tag("TENSORS").packets.push_back(
Adopt(tensors.release()).At(Timestamp(0)));
}
const Tensor& GetOutput() {
return runner_.Outputs()
.Get("TENSORS", 0)
.packets[0]
.Get<std::vector<Tensor>>()[0];
}
CalculatorRunner runner_;
};
TEST_F(TensorsDequantizationCalculatorTest, FailsWithFloatTensors) {
std::vector<float> tensor = {0, 1};
PushTensor(Tensor::ElementType::kFloat32, tensor);
auto status = runner_.Run();
EXPECT_EQ(status.code(), absl::StatusCode::kInvalidArgument);
EXPECT_THAT(status.message(), HasSubstr("Unsupported input tensor type"));
}
TEST_F(TensorsDequantizationCalculatorTest, FailsWithInt32Tensors) {
std::vector<int32_t> tensor = {0, 1};
PushTensor(Tensor::ElementType::kInt32, tensor);
auto status = runner_.Run();
EXPECT_EQ(status.code(), absl::StatusCode::kInvalidArgument);
EXPECT_THAT(status.message(), HasSubstr("Unsupported input tensor type"));
}
TEST_F(TensorsDequantizationCalculatorTest, SucceedsWithUInt8Tensors) {
std::vector<uint8_t> tensor = {0, 127, 255};
PushTensor(Tensor::ElementType::kUInt8, tensor,
Tensor::QuantizationParameters{1.0f / 127, 127});
MP_ASSERT_OK(runner_.Run());
ValidateResult(GetOutput(), {-1, 0, 1.007874});
}
TEST_F(TensorsDequantizationCalculatorTest, SucceedsWithInt8Tensors) {
std::vector<int8_t> tensor = {-128, 0, 127};
PushTensor(Tensor::ElementType::kInt8, tensor,
Tensor::QuantizationParameters{1.0f / 127, 0});
MP_ASSERT_OK(runner_.Run());
ValidateResult(GetOutput(), {-1.007874, 0, 1});
}
} // namespace
} // namespace mediapipe
mediapipe/calculators/tensor/tensors_to_classification_calculator.cc
+1
View File
@@ -165,6 +165,7 @@ absl::Status TensorsToClassificationCalculator::Open(CalculatorContext* cc) {
absl::Status TensorsToClassificationCalculator::Process(CalculatorContext* cc) {
const auto& input_tensors = *kInTensors(cc);
RET_CHECK_EQ(input_tensors.size(), 1);
RET_CHECK(input_tensors[0].element_type() == Tensor::ElementType::kFloat32);
int num_classes = input_tensors[0].shape().num_elements();
mediapipe/calculators/tensor/tensors_to_detections_calculator.cc
+5 -1
View File
@@ -287,7 +287,11 @@ absl::Status TensorsToDetectionsCalculator::Process(CalculatorContext* cc) {
}
}
}
const int num_input_tensors = kInTensors(cc)->size();
const auto& input_tensors = *kInTensors(cc);
for (const auto& tensor : input_tensors) {
RET_CHECK(tensor.element_type() == Tensor::ElementType::kFloat32);
}
const int num_input_tensors = input_tensors.size();
if (!scores_tensor_index_is_set_) {
if (num_input_tensors == 2 ||
num_input_tensors == kNumInputTensorsWithAnchors) {
mediapipe/calculators/tensor/tensors_to_floats_calculator.cc
+1
View File
@@ -76,6 +76,7 @@ absl::Status TensorsToFloatsCalculator::Open(CalculatorContext* cc) {
absl::Status TensorsToFloatsCalculator::Process(CalculatorContext* cc) {
const auto& input_tensors = *kInTensors(cc);
RET_CHECK(!input_tensors.empty());
RET_CHECK(input_tensors[0].element_type() == Tensor::ElementType::kFloat32);
// TODO: Add option to specify which tensor to take from.
auto view = input_tensors[0].GetCpuReadView();
auto raw_floats = view.buffer<float>();
mediapipe/calculators/tensor/tensors_to_landmarks_calculator.cc
+1
View File
@@ -139,6 +139,7 @@ absl::Status TensorsToLandmarksCalculator::Process(CalculatorContext* cc) {
bool flip_vertically = kFlipVertically(cc).GetOr(options_.flip_vertically());
const auto& input_tensors = *kInTensors(cc);
RET_CHECK(input_tensors[0].element_type() == Tensor::ElementType::kFloat32);
int num_values = input_tensors[0].shape().num_elements();
const int num_dimensions = num_values / num_landmarks_;
CHECK_GT(num_dimensions, 0);
mediapipe/calculators/tensor/tensors_to_segmentation_calculator.cc
+4 -2
View File
@@ -116,8 +116,9 @@ using ::tflite::gpu::gl::GlShader;
//
// Inputs:
// One of the following TENSORS tags:
// TENSORS: Vector of Tensor,
// The tensor dimensions are specified in this calculator's options.
// TENSORS: Vector of Tensors of type kFloat32. Only the first tensor will be
// used. The tensor dimensions are specified in this calculator's
// options.
// OUTPUT_SIZE(optional): std::pair<int, int>,
// If provided, the size to upscale mask to.
//
@@ -261,6 +262,7 @@ absl::Status TensorsToSegmentationCalculator::Process(CalculatorContext* cc) {
// Validate tensor channels and activation type.
{
RET_CHECK(!input_tensors.empty());
RET_CHECK(input_tensors[0].element_type() == Tensor::ElementType::kFloat32);
ASSIGN_OR_RETURN(auto hwc, GetHwcFromDims(input_tensors[0].shape().dims));
int tensor_channels = std::get<2>(hwc);
typedef mediapipe::TensorsToSegmentationCalculatorOptions Options;