talksik/mediapipe
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Project import generated by Copybara.

Browse Source 
GitOrigin-RevId: d8caa66de45839696f5bd0786ad3bfbcb9cff632
This commit is contained in:
MediaPipe Team
2020-12-09 22:43:33 -05:00
committed by chuoling
parent f15da632de
commit 2b58cceec9
750 changed files with 22901 additions and 9478 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mediapipe/MediaPipe.tulsiproj/Configs/MediaPipe.tulsigen
+6
View File
@@ -14,9 +14,11 @@
"mediapipe/examples/ios/facemeshgpu/BUILD",
"mediapipe/examples/ios/handdetectiongpu/BUILD",
"mediapipe/examples/ios/handtrackinggpu/BUILD",
"mediapipe/examples/ios/holistictrackinggpu/BUILD",
"mediapipe/examples/ios/iristrackinggpu/BUILD",
"mediapipe/examples/ios/objectdetectioncpu/BUILD",
"mediapipe/examples/ios/objectdetectiongpu/BUILD",
"mediapipe/examples/ios/posetrackinggpu/BUILD",
"mediapipe/examples/ios/upperbodyposetrackinggpu/BUILD"
],
"buildTargets" : [
@@ -27,9 +29,11 @@
"//mediapipe/examples/ios/facemeshgpu:FaceMeshGpuApp",
"//mediapipe/examples/ios/handdetectiongpu:HandDetectionGpuApp",
"//mediapipe/examples/ios/handtrackinggpu:HandTrackingGpuApp",
"//mediapipe/examples/ios/holistictrackinggpu:HolisticTrackingGpuApp",
"//mediapipe/examples/ios/iristrackinggpu:IrisTrackingGpuApp",
"//mediapipe/examples/ios/objectdetectioncpu:ObjectDetectionCpuApp",
"//mediapipe/examples/ios/objectdetectiongpu:ObjectDetectionGpuApp",
"//mediapipe/examples/ios/posetrackinggpu:PoseTrackingGpuApp",
"//mediapipe/examples/ios/upperbodyposetrackinggpu:UpperBodyPoseTrackingGpuApp",
"//mediapipe/objc:mediapipe_framework_ios"
],
@@ -94,9 +98,11 @@
"mediapipe/examples/ios/faceeffect/Base.lproj",
"mediapipe/examples/ios/handdetectiongpu",
"mediapipe/examples/ios/handtrackinggpu",
"mediapipe/examples/ios/holistictrackinggpu",
"mediapipe/examples/ios/iristrackinggpu",
"mediapipe/examples/ios/objectdetectioncpu",
"mediapipe/examples/ios/objectdetectiongpu",
"mediapipe/examples/ios/posetrackinggpu",
"mediapipe/examples/ios/upperbodyposetrackinggpu",
"mediapipe/framework",
"mediapipe/framework/deps",
mediapipe/MediaPipe.tulsiproj/project.tulsiconf
+2
View File
@@ -17,9 +17,11 @@
"mediapipe/examples/ios/facemeshgpu",
"mediapipe/examples/ios/handdetectiongpu",
"mediapipe/examples/ios/handtrackinggpu",
"mediapipe/examples/ios/holistictrackinggpu",
"mediapipe/examples/ios/iristrackinggpu",
"mediapipe/examples/ios/objectdetectioncpu",
"mediapipe/examples/ios/objectdetectiongpu",
"mediapipe/examples/ios/posetrackinggpu",
"mediapipe/examples/ios/upperbodyposetrackinggpu"
],
"projectName" : "Mediapipe",
mediapipe/calculators/audio/audio_decoder_calculator.cc
+10 -11
View File
@@ -48,18 +48,17 @@ namespace mediapipe {
// TODO: support decoding multiple streams.
class AudioDecoderCalculator : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc);
static mediapipe::Status GetContract(CalculatorContract* cc);
::mediapipe::Status Open(CalculatorContext* cc) override;
::mediapipe::Status Process(CalculatorContext* cc) override;
::mediapipe::Status Close(CalculatorContext* cc) override;
mediapipe::Status Open(CalculatorContext* cc) override;
mediapipe::Status Process(CalculatorContext* cc) override;
mediapipe::Status Close(CalculatorContext* cc) override;
private:
std::unique_ptr<AudioDecoder> decoder_;
};
::mediapipe::Status AudioDecoderCalculator::GetContract(
CalculatorContract* cc) {
mediapipe::Status AudioDecoderCalculator::GetContract(CalculatorContract* cc) {
cc->InputSidePackets().Tag("INPUT_FILE_PATH").Set<std::string>();
if (cc->InputSidePackets().HasTag("OPTIONS")) {
cc->InputSidePackets().Tag("OPTIONS").Set<mediapipe::AudioDecoderOptions>();
@@ -68,10 +67,10 @@ class AudioDecoderCalculator : public CalculatorBase {
if (cc->Outputs().HasTag("AUDIO_HEADER")) {
cc->Outputs().Tag("AUDIO_HEADER").SetNone();
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status AudioDecoderCalculator::Open(CalculatorContext* cc) {
mediapipe::Status AudioDecoderCalculator::Open(CalculatorContext* cc) {
const std::string& input_file_path =
cc->InputSidePackets().Tag("INPUT_FILE_PATH").Get<std::string>();
const auto& decoder_options =
@@ -88,10 +87,10 @@ class AudioDecoderCalculator : public CalculatorBase {
cc->Outputs().Tag("AUDIO_HEADER").SetHeader(Adopt(header.release()));
}
cc->Outputs().Tag("AUDIO_HEADER").Close();
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status AudioDecoderCalculator::Process(CalculatorContext* cc) {
mediapipe::Status AudioDecoderCalculator::Process(CalculatorContext* cc) {
Packet data;
int options_index = -1;
auto status = decoder_->GetData(&options_index, &data);
@@ -101,7 +100,7 @@ class AudioDecoderCalculator : public CalculatorBase {
return status;
}
::mediapipe::Status AudioDecoderCalculator::Close(CalculatorContext* cc) {
mediapipe::Status AudioDecoderCalculator::Close(CalculatorContext* cc) {
return decoder_->Close();
}
mediapipe/calculators/audio/basic_time_series_calculators.cc
+26 -26
View File
@@ -38,7 +38,7 @@ static bool SafeMultiply(int x, int y, int* result) {
}
} // namespace
::mediapipe::Status BasicTimeSeriesCalculatorBase::GetContract(
mediapipe::Status BasicTimeSeriesCalculatorBase::GetContract(
CalculatorContract* cc) {
cc->Inputs().Index(0).Set<Matrix>(
// Input stream with TimeSeriesHeader.
@@ -46,10 +46,10 @@ static bool SafeMultiply(int x, int y, int* result) {
cc->Outputs().Index(0).Set<Matrix>(
// Output stream with TimeSeriesHeader.
);
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status BasicTimeSeriesCalculatorBase::Open(CalculatorContext* cc) {
mediapipe::Status BasicTimeSeriesCalculatorBase::Open(CalculatorContext* cc) {
TimeSeriesHeader input_header;
MP_RETURN_IF_ERROR(time_series_util::FillTimeSeriesHeaderIfValid(
cc->Inputs().Index(0).Header(), &input_header));
@@ -57,10 +57,10 @@ static bool SafeMultiply(int x, int y, int* result) {
auto output_header = new TimeSeriesHeader(input_header);
MP_RETURN_IF_ERROR(MutateHeader(output_header));
cc->Outputs().Index(0).SetHeader(Adopt(output_header));
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status BasicTimeSeriesCalculatorBase::Process(
mediapipe::Status BasicTimeSeriesCalculatorBase::Process(
CalculatorContext* cc) {
const Matrix& input = cc->Inputs().Index(0).Get<Matrix>();
MP_RETURN_IF_ERROR(time_series_util::IsMatrixShapeConsistentWithHeader(
@@ -71,12 +71,12 @@ static bool SafeMultiply(int x, int y, int* result) {
*output, cc->Outputs().Index(0).Header().Get<TimeSeriesHeader>()));
cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp());
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status BasicTimeSeriesCalculatorBase::MutateHeader(
mediapipe::Status BasicTimeSeriesCalculatorBase::MutateHeader(
TimeSeriesHeader* output_header) {
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
// Calculator to sum an input time series across channels. This is
@@ -86,9 +86,9 @@ static bool SafeMultiply(int x, int y, int* result) {
class SumTimeSeriesAcrossChannelsCalculator
: public BasicTimeSeriesCalculatorBase {
protected:
::mediapipe::Status MutateHeader(TimeSeriesHeader* output_header) final {
mediapipe::Status MutateHeader(TimeSeriesHeader* output_header) final {
output_header->set_num_channels(1);
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
Matrix ProcessMatrix(const Matrix& input_matrix) final {
@@ -104,9 +104,9 @@ REGISTER_CALCULATOR(SumTimeSeriesAcrossChannelsCalculator);
class AverageTimeSeriesAcrossChannelsCalculator
: public BasicTimeSeriesCalculatorBase {
protected:
::mediapipe::Status MutateHeader(TimeSeriesHeader* output_header) final {
mediapipe::Status MutateHeader(TimeSeriesHeader* output_header) final {
output_header->set_num_channels(1);
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
Matrix ProcessMatrix(const Matrix& input_matrix) final {
@@ -122,7 +122,7 @@ REGISTER_CALCULATOR(AverageTimeSeriesAcrossChannelsCalculator);
// Options proto: None.
class SummarySaiToPitchogramCalculator : public BasicTimeSeriesCalculatorBase {
protected:
::mediapipe::Status MutateHeader(TimeSeriesHeader* output_header) final {
mediapipe::Status MutateHeader(TimeSeriesHeader* output_header) final {
if (output_header->num_channels() != 1) {
return tool::StatusInvalid(
absl::StrCat("Expected single-channel input, got ",
@@ -131,7 +131,7 @@ class SummarySaiToPitchogramCalculator : public BasicTimeSeriesCalculatorBase {
output_header->set_num_channels(output_header->num_samples());
output_header->set_num_samples(1);
output_header->set_sample_rate(output_header->packet_rate());
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
Matrix ProcessMatrix(const Matrix& input_matrix) final {
@@ -160,7 +160,7 @@ REGISTER_CALCULATOR(ReverseChannelOrderCalculator);
// Options proto: None.
class FlattenPacketCalculator : public BasicTimeSeriesCalculatorBase {
protected:
::mediapipe::Status MutateHeader(TimeSeriesHeader* output_header) final {
mediapipe::Status MutateHeader(TimeSeriesHeader* output_header) final {
const int num_input_channels = output_header->num_channels();
const int num_input_samples = output_header->num_samples();
RET_CHECK(num_input_channels >= 0)
@@ -174,7 +174,7 @@ class FlattenPacketCalculator : public BasicTimeSeriesCalculatorBase {
output_header->set_num_channels(output_num_channels);
output_header->set_num_samples(1);
output_header->set_sample_rate(output_header->packet_rate());
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
Matrix ProcessMatrix(const Matrix& input_matrix) final {
@@ -253,10 +253,10 @@ REGISTER_CALCULATOR(DivideByMeanAcrossChannelsCalculator);
// Options proto: None.
class MeanCalculator : public BasicTimeSeriesCalculatorBase {
protected:
::mediapipe::Status MutateHeader(TimeSeriesHeader* output_header) final {
mediapipe::Status MutateHeader(TimeSeriesHeader* output_header) final {
output_header->set_num_samples(1);
output_header->set_sample_rate(output_header->packet_rate());
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
Matrix ProcessMatrix(const Matrix& input_matrix) final {
@@ -272,10 +272,10 @@ REGISTER_CALCULATOR(MeanCalculator);
// Options proto: None.
class StandardDeviationCalculator : public BasicTimeSeriesCalculatorBase {
protected:
::mediapipe::Status MutateHeader(TimeSeriesHeader* output_header) final {
mediapipe::Status MutateHeader(TimeSeriesHeader* output_header) final {
output_header->set_num_samples(1);
output_header->set_sample_rate(output_header->packet_rate());
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
Matrix ProcessMatrix(const Matrix& input_matrix) final {
@@ -293,9 +293,9 @@ REGISTER_CALCULATOR(StandardDeviationCalculator);
// Options proto: None.
class CovarianceCalculator : public BasicTimeSeriesCalculatorBase {
protected:
::mediapipe::Status MutateHeader(TimeSeriesHeader* output_header) final {
mediapipe::Status MutateHeader(TimeSeriesHeader* output_header) final {
output_header->set_num_samples(output_header->num_channels());
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
Matrix ProcessMatrix(const Matrix& input_matrix) final {
@@ -313,9 +313,9 @@ REGISTER_CALCULATOR(CovarianceCalculator);
// Options proto: None.
class L2NormCalculator : public BasicTimeSeriesCalculatorBase {
protected:
::mediapipe::Status MutateHeader(TimeSeriesHeader* output_header) final {
mediapipe::Status MutateHeader(TimeSeriesHeader* output_header) final {
output_header->set_num_channels(1);
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
Matrix ProcessMatrix(const Matrix& input_matrix) final {
@@ -385,12 +385,12 @@ REGISTER_CALCULATOR(ElementwiseSquareCalculator);
// Options proto: None.
class FirstHalfSlicerCalculator : public BasicTimeSeriesCalculatorBase {
protected:
::mediapipe::Status MutateHeader(TimeSeriesHeader* output_header) final {
mediapipe::Status MutateHeader(TimeSeriesHeader* output_header) final {
const int num_input_samples = output_header->num_samples();
RET_CHECK(num_input_samples >= 0)
<< "FirstHalfSlicerCalculator: num_input_samples < 0";
output_header->set_num_samples(num_input_samples / 2);
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
Matrix ProcessMatrix(const Matrix& input_matrix) final {
mediapipe/calculators/audio/basic_time_series_calculators.h
+4 -4
View File
@@ -28,16 +28,16 @@ namespace mediapipe {
class BasicTimeSeriesCalculatorBase : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc);
::mediapipe::Status Open(CalculatorContext* cc) override;
::mediapipe::Status Process(CalculatorContext* cc) override;
static mediapipe::Status GetContract(CalculatorContract* cc);
mediapipe::Status Open(CalculatorContext* cc) override;
mediapipe::Status Process(CalculatorContext* cc) override;
protected:
// Open() calls this method to mutate the output stream header. The input
// to this function will contain a copy of the input stream header, so
// subclasses that do not need to mutate the header do not need to override
// it.
virtual ::mediapipe::Status MutateHeader(TimeSeriesHeader* output_header);
virtual mediapipe::Status MutateHeader(TimeSeriesHeader* output_header);
// Process() calls this method on each packet to compute the output matrix.
virtual Matrix ProcessMatrix(const Matrix& input_matrix) = 0;
mediapipe/calculators/audio/mfcc_mel_calculators.cc
+24 -24
View File
@@ -66,7 +66,7 @@ std::string PortableDebugString(const TimeSeriesHeader& header) {
// rows corresponding to the new feature space).
class FramewiseTransformCalculatorBase : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
static mediapipe::Status GetContract(CalculatorContract* cc) {
cc->Inputs().Index(0).Set<Matrix>(
// Sequence of Matrices, each column describing a particular time frame,
// each row a feature dimension, with TimeSeriesHeader.
@@ -75,11 +75,11 @@ class FramewiseTransformCalculatorBase : public CalculatorBase {
// Sequence of Matrices, each column describing a particular time frame,
// each row a feature dimension, with TimeSeriesHeader.
);
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status Open(CalculatorContext* cc) override;
::mediapipe::Status Process(CalculatorContext* cc) override;
mediapipe::Status Open(CalculatorContext* cc) override;
mediapipe::Status Process(CalculatorContext* cc) override;
int num_output_channels(void) { return num_output_channels_; }
@@ -90,8 +90,8 @@ class FramewiseTransformCalculatorBase : public CalculatorBase {
private:
// Takes header and options, and sets up state including calling
// set_num_output_channels() on the base object.
virtual ::mediapipe::Status ConfigureTransform(const TimeSeriesHeader& header,
CalculatorContext* cc) = 0;
virtual mediapipe::Status ConfigureTransform(const TimeSeriesHeader& header,
CalculatorContext* cc) = 0;
// Takes a vector<double> corresponding to an input frame, and
// perform the specific transformation to produce an output frame.
@@ -102,13 +102,13 @@ class FramewiseTransformCalculatorBase : public CalculatorBase {
int num_output_channels_;
};
::mediapipe::Status FramewiseTransformCalculatorBase::Open(
mediapipe::Status FramewiseTransformCalculatorBase::Open(
CalculatorContext* cc) {
TimeSeriesHeader input_header;
MP_RETURN_IF_ERROR(time_series_util::FillTimeSeriesHeaderIfValid(
cc->Inputs().Index(0).Header(), &input_header));
::mediapipe::Status status = ConfigureTransform(input_header, cc);
mediapipe::Status status = ConfigureTransform(input_header, cc);
auto output_header = new TimeSeriesHeader(input_header);
output_header->set_num_channels(num_output_channels_);
@@ -117,7 +117,7 @@ class FramewiseTransformCalculatorBase : public CalculatorBase {
return status;
}
::mediapipe::Status FramewiseTransformCalculatorBase::Process(
mediapipe::Status FramewiseTransformCalculatorBase::Process(
CalculatorContext* cc) {
const Matrix& input = cc->Inputs().Index(0).Get<Matrix>();
const int num_frames = input.cols();
@@ -145,7 +145,7 @@ class FramewiseTransformCalculatorBase : public CalculatorBase {
}
cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp());
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
// Calculator wrapper around the dsp/mfcc/mfcc.cc routine.
@@ -170,13 +170,13 @@ class FramewiseTransformCalculatorBase : public CalculatorBase {
// }
class MfccCalculator : public FramewiseTransformCalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
static mediapipe::Status GetContract(CalculatorContract* cc) {
return FramewiseTransformCalculatorBase::GetContract(cc);
}
private:
::mediapipe::Status ConfigureTransform(const TimeSeriesHeader& header,
CalculatorContext* cc) override {
mediapipe::Status ConfigureTransform(const TimeSeriesHeader& header,
CalculatorContext* cc) override {
MfccCalculatorOptions mfcc_options = cc->Options<MfccCalculatorOptions>();
mfcc_.reset(new audio_dsp::Mfcc());
int input_length = header.num_channels();
@@ -194,7 +194,7 @@ class MfccCalculator : public FramewiseTransformCalculatorBase {
// audio_dsp::MelFilterBank needs to know this to
// correctly interpret the spectrogram bins.
if (!header.has_audio_sample_rate()) {
return ::mediapipe::InvalidArgumentError(
return mediapipe::InvalidArgumentError(
absl::StrCat("No audio_sample_rate in input TimeSeriesHeader ",
PortableDebugString(header)));
}
@@ -203,10 +203,10 @@ class MfccCalculator : public FramewiseTransformCalculatorBase {
mfcc_->Initialize(input_length, header.audio_sample_rate());
if (initialized) {
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
} else {
return ::mediapipe::Status(mediapipe::StatusCode::kInternal,
"Mfcc::Initialize returned uninitialized");
return mediapipe::Status(mediapipe::StatusCode::kInternal,
"Mfcc::Initialize returned uninitialized");
}
}
@@ -228,13 +228,13 @@ REGISTER_CALCULATOR(MfccCalculator);
// if you ask for too many channels.
class MelSpectrumCalculator : public FramewiseTransformCalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
static mediapipe::Status GetContract(CalculatorContract* cc) {
return FramewiseTransformCalculatorBase::GetContract(cc);
}
private:
::mediapipe::Status ConfigureTransform(const TimeSeriesHeader& header,
CalculatorContext* cc) override {
mediapipe::Status ConfigureTransform(const TimeSeriesHeader& header,
CalculatorContext* cc) override {
MelSpectrumCalculatorOptions mel_spectrum_options =
cc->Options<MelSpectrumCalculatorOptions>();
mel_filterbank_.reset(new audio_dsp::MelFilterbank());
@@ -245,7 +245,7 @@ class MelSpectrumCalculator : public FramewiseTransformCalculatorBase {
// audio_dsp::MelFilterBank needs to know this to
// correctly interpret the spectrogram bins.
if (!header.has_audio_sample_rate()) {
return ::mediapipe::InvalidArgumentError(
return mediapipe::InvalidArgumentError(
absl::StrCat("No audio_sample_rate in input TimeSeriesHeader ",
PortableDebugString(header)));
}
@@ -255,10 +255,10 @@ class MelSpectrumCalculator : public FramewiseTransformCalculatorBase {
mel_spectrum_options.max_frequency_hertz());
if (initialized) {
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
} else {
return ::mediapipe::Status(mediapipe::StatusCode::kInternal,
"mfcc::Initialize returned uninitialized");
return mediapipe::Status(mediapipe::StatusCode::kInternal,
"mfcc::Initialize returned uninitialized");
}
}
mediapipe/calculators/audio/mfcc_mel_calculators_test.cc
+1 -1
View File
@@ -84,7 +84,7 @@ class FramewiseTransformCalculatorTest
num_samples_per_packet_ = GenerateRandomNonnegInputStream(kNumPackets);
}
::mediapipe::Status Run() { return this->RunGraph(); }
mediapipe::Status Run() { return this->RunGraph(); }
void CheckResults(int expected_num_channels) {
const auto& output_header =
mediapipe/calculators/audio/rational_factor_resample_calculator.cc
+9 -9
View File
@@ -23,15 +23,15 @@ using audio_dsp::RationalFactorResampler;
using audio_dsp::Resampler;
namespace mediapipe {
::mediapipe::Status RationalFactorResampleCalculator::Process(
mediapipe::Status RationalFactorResampleCalculator::Process(
CalculatorContext* cc) {
return ProcessInternal(cc->Inputs().Index(0).Get<Matrix>(), false, cc);
}
::mediapipe::Status RationalFactorResampleCalculator::Close(
mediapipe::Status RationalFactorResampleCalculator::Close(
CalculatorContext* cc) {
if (initial_timestamp_ == Timestamp::Unstarted()) {
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
Matrix empty_input_frame(num_channels_, 0);
return ProcessInternal(empty_input_frame, true, cc);
@@ -62,7 +62,7 @@ void CopyVectorToChannel(const std::vector<float>& vec, Matrix* matrix,
} // namespace
::mediapipe::Status RationalFactorResampleCalculator::Open(
mediapipe::Status RationalFactorResampleCalculator::Open(
CalculatorContext* cc) {
RationalFactorResampleCalculatorOptions resample_options =
cc->Options<RationalFactorResampleCalculatorOptions>();
@@ -88,7 +88,7 @@ void CopyVectorToChannel(const std::vector<float>& vec, Matrix* matrix,
resample_options);
if (!r) {
LOG(ERROR) << "Failed to initialize resampler.";
return ::mediapipe::UnknownError("Failed to initialize resampler.");
return mediapipe::UnknownError("Failed to initialize resampler.");
}
}
}
@@ -106,10 +106,10 @@ void CopyVectorToChannel(const std::vector<float>& vec, Matrix* matrix,
initial_timestamp_ = Timestamp::Unstarted();
check_inconsistent_timestamps_ =
resample_options.check_inconsistent_timestamps();
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status RationalFactorResampleCalculator::ProcessInternal(
mediapipe::Status RationalFactorResampleCalculator::ProcessInternal(
const Matrix& input_frame, bool should_flush, CalculatorContext* cc) {
if (initial_timestamp_ == Timestamp::Unstarted()) {
initial_timestamp_ = cc->InputTimestamp();
@@ -131,7 +131,7 @@ void CopyVectorToChannel(const std::vector<float>& vec, Matrix* matrix,
*output_frame = input_frame;
} else {
if (!Resample(input_frame, output_frame.get(), should_flush)) {
return ::mediapipe::UnknownError("Resample() failed.");
return mediapipe::UnknownError("Resample() failed.");
}
}
cumulative_output_samples_ += output_frame->cols();
@@ -139,7 +139,7 @@ void CopyVectorToChannel(const std::vector<float>& vec, Matrix* matrix,
if (output_frame->cols() > 0) {
cc->Outputs().Index(0).Add(output_frame.release(), output_timestamp);
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
bool RationalFactorResampleCalculator::Resample(const Matrix& input_frame,
mediapipe/calculators/audio/rational_factor_resample_calculator.h
+7 -7
View File
@@ -40,24 +40,24 @@ class RationalFactorResampleCalculator : public CalculatorBase {
public:
struct TestAccess;
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
static mediapipe::Status GetContract(CalculatorContract* cc) {
cc->Inputs().Index(0).Set<Matrix>(
// Single input stream with TimeSeriesHeader.
);
cc->Outputs().Index(0).Set<Matrix>(
// Resampled stream with TimeSeriesHeader.
);
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
// Returns FAIL if the input stream header is invalid or if the
// resampler cannot be initialized.
::mediapipe::Status Open(CalculatorContext* cc) override;
mediapipe::Status Open(CalculatorContext* cc) override;
// Resamples a packet of TimeSeries data. Returns FAIL if the
// resampler state becomes inconsistent.
::mediapipe::Status Process(CalculatorContext* cc) override;
mediapipe::Status Process(CalculatorContext* cc) override;
// Flushes any remaining state. Returns FAIL if the resampler state
// becomes inconsistent.
::mediapipe::Status Close(CalculatorContext* cc) override;
mediapipe::Status Close(CalculatorContext* cc) override;
protected:
typedef audio_dsp::Resampler<float> ResamplerType;
@@ -72,8 +72,8 @@ class RationalFactorResampleCalculator : public CalculatorBase {
// Does Timestamp bookkeeping and resampling common to Process() and
// Close(). Returns FAIL if the resampler state becomes
// inconsistent.
::mediapipe::Status ProcessInternal(const Matrix& input_frame,
bool should_flush, CalculatorContext* cc);
mediapipe::Status ProcessInternal(const Matrix& input_frame,
bool should_flush, CalculatorContext* cc);
// Uses the internal resampler_ objects to actually resample each
// row of the input TimeSeries. Returns false if the resampler
mediapipe/calculators/audio/rational_factor_resample_calculator_test.cc
+1 -1
View File
@@ -80,7 +80,7 @@ class RationalFactorResampleCalculatorTest
}
// Initializes and runs the test graph.
::mediapipe::Status Run(double output_sample_rate) {
mediapipe::Status Run(double output_sample_rate) {
options_.set_target_sample_rate(output_sample_rate);
InitializeGraph();
mediapipe/calculators/audio/spectrogram_calculator.cc
+20 -20
View File
@@ -66,7 +66,7 @@ namespace mediapipe {
// analysis frame will advance from its predecessor by the same time step.
class SpectrogramCalculator : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
static mediapipe::Status GetContract(CalculatorContract* cc) {
cc->Inputs().Index(0).Set<Matrix>(
// Input stream with TimeSeriesHeader.
);
@@ -96,21 +96,21 @@ class SpectrogramCalculator : public CalculatorBase {
);
}
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
// Returns FAIL if the input stream header is invalid.
::mediapipe::Status Open(CalculatorContext* cc) override;
mediapipe::Status Open(CalculatorContext* cc) override;
// Outputs at most one packet consisting of a single Matrix with one or
// more columns containing the spectral values from as many input frames
// as are completed by the input samples. Always returns OK.
::mediapipe::Status Process(CalculatorContext* cc) override;
mediapipe::Status Process(CalculatorContext* cc) override;
// Performs zero-padding and processing of any remaining samples
// if pad_final_packet is set.
// Returns OK.
::mediapipe::Status Close(CalculatorContext* cc) override;
mediapipe::Status Close(CalculatorContext* cc) override;
private:
Timestamp CurrentOutputTimestamp(CalculatorContext* cc) {
@@ -138,12 +138,12 @@ class SpectrogramCalculator : public CalculatorBase {
// Convert the output of the spectrogram object into a Matrix (or an
// Eigen::MatrixXcf if complex-valued output is requested) and pass to
// MediaPipe output.
::mediapipe::Status ProcessVector(const Matrix& input_stream,
CalculatorContext* cc);
mediapipe::Status ProcessVector(const Matrix& input_stream,
CalculatorContext* cc);
// Templated function to process either real- or complex-output spectrogram.
template <class OutputMatrixType>
::mediapipe::Status ProcessVectorToOutput(
mediapipe::Status ProcessVectorToOutput(
const Matrix& input_stream,
const OutputMatrixType postprocess_output_fn(const OutputMatrixType&),
CalculatorContext* cc);
@@ -177,7 +177,7 @@ REGISTER_CALCULATOR(SpectrogramCalculator);
// Factor to convert ln(magnitude_squared) to deciBels = 10.0/ln(10.0).
const float SpectrogramCalculator::kLnPowerToDb = 4.342944819032518;
::mediapipe::Status SpectrogramCalculator::Open(CalculatorContext* cc) {
mediapipe::Status SpectrogramCalculator::Open(CalculatorContext* cc) {
SpectrogramCalculatorOptions spectrogram_options =
cc->Options<SpectrogramCalculatorOptions>();
@@ -272,10 +272,10 @@ const float SpectrogramCalculator::kLnPowerToDb = 4.342944819032518;
}
cumulative_completed_frames_ = 0;
initial_input_timestamp_ = Timestamp::Unstarted();
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status SpectrogramCalculator::Process(CalculatorContext* cc) {
mediapipe::Status SpectrogramCalculator::Process(CalculatorContext* cc) {
if (initial_input_timestamp_ == Timestamp::Unstarted()) {
initial_input_timestamp_ = cc->InputTimestamp();
}
@@ -291,7 +291,7 @@ const float SpectrogramCalculator::kLnPowerToDb = 4.342944819032518;
}
template <class OutputMatrixType>
::mediapipe::Status SpectrogramCalculator::ProcessVectorToOutput(
mediapipe::Status SpectrogramCalculator::ProcessVectorToOutput(
const Matrix& input_stream,
const OutputMatrixType postprocess_output_fn(const OutputMatrixType&),
CalculatorContext* cc) {
@@ -311,8 +311,8 @@ template <class OutputMatrixType>
if (!spectrogram_generators_[channel]->ComputeSpectrogram(
input_vector, &output_vectors)) {
return ::mediapipe::Status(mediapipe::StatusCode::kInternal,
"Spectrogram returned failure");
return mediapipe::Status(mediapipe::StatusCode::kInternal,
"Spectrogram returned failure");
}
if (channel == 0) {
// Record the number of time frames we expect from each channel.
@@ -355,10 +355,10 @@ template <class OutputMatrixType>
}
cumulative_completed_frames_ += output_vectors.size();
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status SpectrogramCalculator::ProcessVector(
mediapipe::Status SpectrogramCalculator::ProcessVector(
const Matrix& input_stream, CalculatorContext* cc) {
switch (output_type_) {
// These blocks deliberately ignore clang-format to preserve the
@@ -394,13 +394,13 @@ template <class OutputMatrixType>
}
// clang-format on
default: {
return ::mediapipe::Status(mediapipe::StatusCode::kInvalidArgument,
"Unrecognized spectrogram output type.");
return mediapipe::Status(mediapipe::StatusCode::kInvalidArgument,
"Unrecognized spectrogram output type.");
}
}
}
::mediapipe::Status SpectrogramCalculator::Close(CalculatorContext* cc) {
mediapipe::Status SpectrogramCalculator::Close(CalculatorContext* cc) {
if (cumulative_input_samples_ > 0 && pad_final_packet_) {
// We can flush any remaining samples by sending frame_step_samples - 1
// zeros to the Process method, and letting it do its thing,
@@ -416,7 +416,7 @@ template <class OutputMatrixType>
Matrix::Zero(num_input_channels_, required_padding_samples), cc);
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
} // namespace mediapipe
mediapipe/calculators/audio/spectrogram_calculator_test.cc
+1 -1
View File
@@ -50,7 +50,7 @@ class SpectrogramCalculatorTest
}
// Initializes and runs the test graph.
::mediapipe::Status Run() {
mediapipe::Status Run() {
// Now that options are set, we can set up some internal constants.
frame_duration_samples_ =
round(options_.frame_duration_seconds() * input_sample_rate_);
mediapipe/calculators/audio/stabilized_log_calculator.cc
+8 -8
View File
@@ -41,17 +41,17 @@ namespace mediapipe {
// }
class StabilizedLogCalculator : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
static mediapipe::Status GetContract(CalculatorContract* cc) {
cc->Inputs().Index(0).Set<Matrix>(
// Input stream with TimeSeriesHeader.
);
cc->Outputs().Index(0).Set<Matrix>(
// Output stabilized log stream with TimeSeriesHeader.
);
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status Open(CalculatorContext* cc) override {
mediapipe::Status Open(CalculatorContext* cc) override {
StabilizedLogCalculatorOptions stabilized_log_calculator_options =
cc->Options<StabilizedLogCalculatorOptions>();
@@ -70,23 +70,23 @@ class StabilizedLogCalculator : public CalculatorBase {
cc->Outputs().Index(0).SetHeader(
Adopt(new TimeSeriesHeader(input_header)));
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status Process(CalculatorContext* cc) override {
mediapipe::Status Process(CalculatorContext* cc) override {
auto input_matrix = cc->Inputs().Index(0).Get<Matrix>();
if (input_matrix.array().isNaN().any()) {
return ::mediapipe::InvalidArgumentError("NaN input to log operation.");
return mediapipe::InvalidArgumentError("NaN input to log operation.");
}
if (check_nonnegativity_) {
if (input_matrix.minCoeff() < 0.0) {
return ::mediapipe::OutOfRangeError("Negative input to log operation.");
return mediapipe::OutOfRangeError("Negative input to log operation.");
}
}
std::unique_ptr<Matrix> output_frame(new Matrix(
output_scale_ * (input_matrix.array() + stabilizer_).log().matrix()));
cc->Outputs().Index(0).Add(output_frame.release(), cc->InputTimestamp());
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
private:
mediapipe/calculators/audio/time_series_framer_calculator.cc
+11 -11
View File
@@ -66,26 +66,26 @@ namespace mediapipe {
// cumulative_completed_samples / sample_rate_.
class TimeSeriesFramerCalculator : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
static mediapipe::Status GetContract(CalculatorContract* cc) {
cc->Inputs().Index(0).Set<Matrix>(
// Input stream with TimeSeriesHeader.
);
cc->Outputs().Index(0).Set<Matrix>(
// Fixed length time series Packets with TimeSeriesHeader.
);
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
// Returns FAIL if the input stream header is invalid.
::mediapipe::Status Open(CalculatorContext* cc) override;
mediapipe::Status Open(CalculatorContext* cc) override;
// Outputs as many framed packets as possible given the accumulated
// input. Always returns OK.
::mediapipe::Status Process(CalculatorContext* cc) override;
mediapipe::Status Process(CalculatorContext* cc) override;
// Flushes any remaining samples in a zero-padded packet. Always
// returns OK.
::mediapipe::Status Close(CalculatorContext* cc) override;
mediapipe::Status Close(CalculatorContext* cc) override;
private:
// Adds input data to the internal buffer.
@@ -205,7 +205,7 @@ void TimeSeriesFramerCalculator::FrameOutput(CalculatorContext* cc) {
}
}
::mediapipe::Status TimeSeriesFramerCalculator::Process(CalculatorContext* cc) {
mediapipe::Status TimeSeriesFramerCalculator::Process(CalculatorContext* cc) {
if (initial_input_timestamp_ == Timestamp::Unstarted()) {
initial_input_timestamp_ = cc->InputTimestamp();
current_timestamp_ = initial_input_timestamp_;
@@ -214,10 +214,10 @@ void TimeSeriesFramerCalculator::FrameOutput(CalculatorContext* cc) {
EnqueueInput(cc);
FrameOutput(cc);
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status TimeSeriesFramerCalculator::Close(CalculatorContext* cc) {
mediapipe::Status TimeSeriesFramerCalculator::Close(CalculatorContext* cc) {
while (samples_still_to_drop_ > 0 && !sample_buffer_.empty()) {
sample_buffer_.pop_front();
--samples_still_to_drop_;
@@ -234,10 +234,10 @@ void TimeSeriesFramerCalculator::FrameOutput(CalculatorContext* cc) {
CurrentOutputTimestamp());
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status TimeSeriesFramerCalculator::Open(CalculatorContext* cc) {
mediapipe::Status TimeSeriesFramerCalculator::Open(CalculatorContext* cc) {
TimeSeriesFramerCalculatorOptions framer_options =
cc->Options<TimeSeriesFramerCalculatorOptions>();
@@ -317,7 +317,7 @@ void TimeSeriesFramerCalculator::FrameOutput(CalculatorContext* cc) {
}
use_local_timestamp_ = framer_options.use_local_timestamp();
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
} // namespace mediapipe
mediapipe/calculators/audio/time_series_framer_calculator_test.cc
+2 -2
View File
@@ -69,7 +69,7 @@ class TimeSeriesFramerCalculatorTest
}
// Initializes and runs the test graph.
::mediapipe::Status Run() {
mediapipe::Status Run() {
InitializeGraph();
FillInputHeader();
@@ -441,7 +441,7 @@ class TimeSeriesFramerCalculatorTimestampingTest
}
}
::mediapipe::Status RunTimestampTest() {
mediapipe::Status RunTimestampTest() {
InitializeGraph();
InitializeInputForTimeStampingTest();
FillInputHeader();
mediapipe/calculators/core/BUILD
+16 -1
View File
@@ -130,6 +130,16 @@ mediapipe_proto_library(
],
)
mediapipe_proto_library(
name = "flow_limiter_calculator_proto",
srcs = ["flow_limiter_calculator.proto"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
cc_library(
name = "add_header_calculator",
srcs = ["add_header_calculator.cc"],
@@ -238,13 +248,14 @@ cc_library(
visibility = ["//visibility:public"],
deps = [
":concatenate_vector_calculator_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/formats:classification_cc_proto",
"//mediapipe/framework/formats:landmark_cc_proto",
"//mediapipe/framework/formats:tensor",
"//mediapipe/framework/port:integral_types",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"//mediapipe/framework:calculator_framework",
"//mediapipe/util:render_data_cc_proto",
"@org_tensorflow//tensorflow/lite:framework",
] + select({
"//mediapipe/gpu:disable_gpu": [],
@@ -607,6 +618,7 @@ cc_library(
srcs = ["flow_limiter_calculator.cc"],
visibility = ["//visibility:public"],
deps = [
":flow_limiter_calculator_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:packet",
"//mediapipe/framework:timestamp",
@@ -782,6 +794,7 @@ cc_test(
srcs = ["flow_limiter_calculator_test.cc"],
deps = [
":flow_limiter_calculator",
":flow_limiter_calculator_cc_proto",
"//mediapipe/calculators/core:counting_source_calculator",
"//mediapipe/calculators/core:pass_through_calculator",
"//mediapipe/framework:calculator_framework",
@@ -793,6 +806,8 @@ cc_test(
"//mediapipe/framework/port:integral_types",
"//mediapipe/framework/port:parse_text_proto",
"//mediapipe/framework/stream_handler:immediate_input_stream_handler",
"//mediapipe/framework/tool:simulation_clock",
"//mediapipe/framework/tool:simulation_clock_executor",
"//mediapipe/framework/tool:sink",
"@com_google_absl//absl/time",
],
mediapipe/calculators/core/add_header_calculator.cc
+6 -6
View File
@@ -44,7 +44,7 @@ namespace mediapipe {
//
class AddHeaderCalculator : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
static mediapipe::Status GetContract(CalculatorContract* cc) {
bool has_side_input = false;
bool has_header_stream = false;
if (cc->InputSidePackets().HasTag("HEADER")) {
@@ -62,10 +62,10 @@ class AddHeaderCalculator : public CalculatorBase {
}
cc->Inputs().Tag("DATA").SetAny();
cc->Outputs().Index(0).SetSameAs(&cc->Inputs().Tag("DATA"));
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status Open(CalculatorContext* cc) override {
mediapipe::Status Open(CalculatorContext* cc) override {
Packet header;
if (cc->InputSidePackets().HasTag("HEADER")) {
header = cc->InputSidePackets().Tag("HEADER");
@@ -77,12 +77,12 @@ class AddHeaderCalculator : public CalculatorBase {
cc->Outputs().Index(0).SetHeader(header);
}
cc->SetOffset(TimestampDiff(0));
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status Process(CalculatorContext* cc) override {
mediapipe::Status Process(CalculatorContext* cc) override {
cc->Outputs().Index(0).AddPacket(cc->Inputs().Tag("DATA").Value());
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
};
mediapipe/calculators/core/add_header_calculator_test.cc
+1 -1
View File
@@ -153,7 +153,7 @@ TEST_F(AddHeaderCalculatorTest, UsingBothSideInputAndStream) {
}
// Run should fail because header can only be provided one way.
EXPECT_EQ(runner.Run().code(), ::mediapipe::InvalidArgumentError("").code());
EXPECT_EQ(runner.Run().code(), mediapipe::InvalidArgumentError("").code());
}
} // namespace mediapipe
mediapipe/calculators/core/begin_end_loop_calculator_graph_test.cc
+18 -18
View File
@@ -42,22 +42,22 @@ REGISTER_CALCULATOR(BeginLoopIntegerCalculator);
class IncrementCalculator : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
static mediapipe::Status GetContract(CalculatorContract* cc) {
cc->Inputs().Index(0).Set<int>();
cc->Outputs().Index(0).Set<int>();
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status Open(CalculatorContext* cc) override {
mediapipe::Status Open(CalculatorContext* cc) override {
cc->SetOffset(TimestampDiff(0));
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status Process(CalculatorContext* cc) override {
mediapipe::Status Process(CalculatorContext* cc) override {
const int& input_int = cc->Inputs().Index(0).Get<int>();
auto output_int = absl::make_unique<int>(input_int + 1);
cc->Outputs().Index(0).Add(output_int.release(), cc->InputTimestamp());
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
};
@@ -166,19 +166,19 @@ TEST_F(BeginEndLoopCalculatorGraphTest, MultipleVectors) {
// bound update.
class PassThroughOrEmptyVectorCalculator : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
static mediapipe::Status GetContract(CalculatorContract* cc) {
cc->SetProcessTimestampBounds(true);
cc->Inputs().Index(0).Set<std::vector<int>>();
cc->Outputs().Index(0).Set<std::vector<int>>();
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status Open(CalculatorContext* cc) override {
mediapipe::Status Open(CalculatorContext* cc) override {
cc->SetOffset(TimestampDiff(0));
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status Process(CalculatorContext* cc) override {
mediapipe::Status Process(CalculatorContext* cc) override {
if (!cc->Inputs().Index(0).IsEmpty()) {
cc->Outputs().Index(0).AddPacket(cc->Inputs().Index(0).Value());
} else {
@@ -186,7 +186,7 @@ class PassThroughOrEmptyVectorCalculator : public CalculatorBase {
MakePacket<std::vector<int>>(std::vector<int>())
.At(cc->InputTimestamp()));
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
};
@@ -311,24 +311,24 @@ TEST_F(BeginEndLoopCalculatorGraphProcessingEmptyPacketsTest, MultipleVectors) {
class MultiplierCalculator : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
static mediapipe::Status GetContract(CalculatorContract* cc) {
cc->Inputs().Index(0).Set<int>();
cc->Inputs().Index(1).Set<int>();
cc->Outputs().Index(0).Set<int>();
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status Open(CalculatorContext* cc) override {
mediapipe::Status Open(CalculatorContext* cc) override {
cc->SetOffset(TimestampDiff(0));
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status Process(CalculatorContext* cc) override {
mediapipe::Status Process(CalculatorContext* cc) override {
const int& input_int = cc->Inputs().Index(0).Get<int>();
const int& multiplier_int = cc->Inputs().Index(1).Get<int>();
auto output_int = absl::make_unique<int>(input_int * multiplier_int);
cc->Outputs().Index(0).Add(output_int.release(), cc->InputTimestamp());
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
};
mediapipe/calculators/core/begin_loop_calculator.h
+4 -4
View File
@@ -61,7 +61,7 @@ class BeginLoopCalculator : public CalculatorBase {
using ItemT = typename IterableT::value_type;
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
static mediapipe::Status GetContract(CalculatorContract* cc) {
// The below enables processing of timestamp bound updates, and that enables
// correct timestamp propagation by the companion EndLoopCalculator.
//
@@ -106,10 +106,10 @@ class BeginLoopCalculator : public CalculatorBase {
}
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status Process(CalculatorContext* cc) final {
mediapipe::Status Process(CalculatorContext* cc) final {
Timestamp last_timestamp = loop_internal_timestamp_;
if (!cc->Inputs().Tag("ITERABLE").IsEmpty()) {
const IterableT& collection =
@@ -139,7 +139,7 @@ class BeginLoopCalculator : public CalculatorBase {
.AddPacket(MakePacket<Timestamp>(cc->InputTimestamp())
.At(Timestamp(loop_internal_timestamp_ - 1)));
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
private:
mediapipe/calculators/core/clip_vector_size_calculator.h
+19 -19
View File
@@ -33,7 +33,7 @@ namespace mediapipe {
// input_stream: "input_vector"
// output_stream: "output_vector"
// options {
// [mediapipe.ClipIntVectorSizeCalculatorOptions.ext] {
// [mediapipe.ClipVectorSizeCalculatorOptions.ext] {
// max_vec_size: 5
// }
// }
@@ -43,13 +43,13 @@ namespace mediapipe {
template <typename T>
class ClipVectorSizeCalculator : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
static mediapipe::Status GetContract(CalculatorContract* cc) {
RET_CHECK(cc->Inputs().NumEntries() == 1);
RET_CHECK(cc->Outputs().NumEntries() == 1);
if (cc->Options<::mediapipe::ClipVectorSizeCalculatorOptions>()
.max_vec_size() < 1) {
return ::mediapipe::InternalError(
return mediapipe::InternalError(
"max_vec_size should be greater than or equal to 1.");
}
@@ -60,10 +60,10 @@ class ClipVectorSizeCalculator : public CalculatorBase {
cc->InputSidePackets().Index(0).Set<int>();
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status Open(CalculatorContext* cc) override {
mediapipe::Status Open(CalculatorContext* cc) override {
cc->SetOffset(TimestampDiff(0));
max_vec_size_ = cc->Options<::mediapipe::ClipVectorSizeCalculatorOptions>()
.max_vec_size();
@@ -72,23 +72,23 @@ class ClipVectorSizeCalculator : public CalculatorBase {
!cc->InputSidePackets().Index(0).IsEmpty()) {
max_vec_size_ = cc->InputSidePackets().Index(0).Get<int>();
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status Process(CalculatorContext* cc) override {
mediapipe::Status Process(CalculatorContext* cc) override {
if (max_vec_size_ < 1) {
return ::mediapipe::InternalError(
return mediapipe::InternalError(
"max_vec_size should be greater than or equal to 1.");
}
if (cc->Inputs().Index(0).IsEmpty()) {
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
return ClipVectorSize<T>(std::is_copy_constructible<T>(), cc);
}
template <typename U>
::mediapipe::Status ClipVectorSize(std::true_type, CalculatorContext* cc) {
mediapipe::Status ClipVectorSize(std::true_type, CalculatorContext* cc) {
auto output = absl::make_unique<std::vector<U>>();
const std::vector<U>& input_vector =
cc->Inputs().Index(0).Get<std::vector<U>>();
@@ -100,19 +100,19 @@ class ClipVectorSizeCalculator : public CalculatorBase {
}
}
cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp());
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
template <typename U>
::mediapipe::Status ClipVectorSize(std::false_type, CalculatorContext* cc) {
mediapipe::Status ClipVectorSize(std::false_type, CalculatorContext* cc) {
return ConsumeAndClipVectorSize<T>(std::is_move_constructible<U>(), cc);
}
template <typename U>
::mediapipe::Status ConsumeAndClipVectorSize(std::true_type,
CalculatorContext* cc) {
mediapipe::Status ConsumeAndClipVectorSize(std::true_type,
CalculatorContext* cc) {
auto output = absl::make_unique<std::vector<U>>();
::mediapipe::StatusOr<std::unique_ptr<std::vector<U>>> input_status =
mediapipe::StatusOr<std::unique_ptr<std::vector<U>>> input_status =
cc->Inputs().Index(0).Value().Consume<std::vector<U>>();
if (input_status.ok()) {
@@ -129,13 +129,13 @@ class ClipVectorSizeCalculator : public CalculatorBase {
return input_status.status();
}
cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp());
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
template <typename U>
::mediapipe::Status ConsumeAndClipVectorSize(std::false_type,
CalculatorContext* cc) {
return ::mediapipe::InternalError(
mediapipe::Status ConsumeAndClipVectorSize(std::false_type,
CalculatorContext* cc) {
return mediapipe::InternalError(
"Cannot copy or move input vectors and clip their size.");
}
mediapipe/calculators/core/concatenate_normalized_landmark_list_calculator.cc
+7 -7
View File
@@ -29,7 +29,7 @@ namespace mediapipe {
// NormalizedLandmarkList proto object.
class ConcatenateNormalizedLandmarkListCalculator : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
static mediapipe::Status GetContract(CalculatorContract* cc) {
RET_CHECK(cc->Inputs().NumEntries() != 0);
RET_CHECK(cc->Outputs().NumEntries() == 1);
@@ -39,21 +39,21 @@ class ConcatenateNormalizedLandmarkListCalculator : public CalculatorBase {
cc->Outputs().Index(0).Set<NormalizedLandmarkList>();
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status Open(CalculatorContext* cc) override {
mediapipe::Status Open(CalculatorContext* cc) override {
cc->SetOffset(TimestampDiff(0));
only_emit_if_all_present_ =
cc->Options<::mediapipe::ConcatenateVectorCalculatorOptions>()
.only_emit_if_all_present();
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status Process(CalculatorContext* cc) override {
mediapipe::Status Process(CalculatorContext* cc) override {
if (only_emit_if_all_present_) {
for (int i = 0; i < cc->Inputs().NumEntries(); ++i) {
if (cc->Inputs().Index(i).IsEmpty()) return ::mediapipe::OkStatus();
if (cc->Inputs().Index(i).IsEmpty()) return mediapipe::OkStatus();
}
}
@@ -69,7 +69,7 @@ class ConcatenateNormalizedLandmarkListCalculator : public CalculatorBase {
}
cc->Outputs().Index(0).AddPacket(
MakePacket<NormalizedLandmarkList>(output).At(cc->InputTimestamp()));
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
private:
mediapipe/calculators/core/concatenate_vector_calculator.cc
+5
View File
@@ -20,6 +20,7 @@
#include "mediapipe/framework/formats/landmark.pb.h"
#include "mediapipe/framework/formats/tensor.h"
#include "mediapipe/framework/port/integral_types.h"
#include "mediapipe/util/render_data.pb.h"
#include "tensorflow/lite/interpreter.h"
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
@@ -86,4 +87,8 @@ typedef ConcatenateVectorCalculator<::tflite::gpu::gl::GlBuffer>
REGISTER_CALCULATOR(ConcatenateGlBufferVectorCalculator);
#endif
typedef ConcatenateVectorCalculator<mediapipe::RenderData>
ConcatenateRenderDataVectorCalculator;
REGISTER_CALCULATOR(ConcatenateRenderDataVectorCalculator);
} // namespace mediapipe
mediapipe/calculators/core/concatenate_vector_calculator.h
+19 -21
View File
@@ -34,7 +34,7 @@ namespace mediapipe {
template <typename T>
class ConcatenateVectorCalculator : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
static mediapipe::Status GetContract(CalculatorContract* cc) {
RET_CHECK(cc->Inputs().NumEntries() != 0);
RET_CHECK(cc->Outputs().NumEntries() == 1);
@@ -45,21 +45,21 @@ class ConcatenateVectorCalculator : public CalculatorBase {
cc->Outputs().Index(0).Set<std::vector<T>>();
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status Open(CalculatorContext* cc) override {
mediapipe::Status Open(CalculatorContext* cc) override {
cc->SetOffset(TimestampDiff(0));
only_emit_if_all_present_ =
cc->Options<::mediapipe::ConcatenateVectorCalculatorOptions>()
.only_emit_if_all_present();
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status Process(CalculatorContext* cc) override {
mediapipe::Status Process(CalculatorContext* cc) override {
if (only_emit_if_all_present_) {
for (int i = 0; i < cc->Inputs().NumEntries(); ++i) {
if (cc->Inputs().Index(i).IsEmpty()) return ::mediapipe::OkStatus();
if (cc->Inputs().Index(i).IsEmpty()) return mediapipe::OkStatus();
}
}
@@ -67,8 +67,7 @@ class ConcatenateVectorCalculator : public CalculatorBase {
}
template <typename U>
::mediapipe::Status ConcatenateVectors(std::true_type,
CalculatorContext* cc) {
mediapipe::Status ConcatenateVectors(std::true_type, CalculatorContext* cc) {
auto output = absl::make_unique<std::vector<U>>();
for (int i = 0; i < cc->Inputs().NumEntries(); ++i) {
auto& input = cc->Inputs().Index(i);
@@ -82,22 +81,21 @@ class ConcatenateVectorCalculator : public CalculatorBase {
const std::vector<U>& value = input.Get<std::vector<U>>();
output->insert(output->end(), value.begin(), value.end());
} else {
return ::mediapipe::InvalidArgumentError("Invalid input stream type.");
return mediapipe::InvalidArgumentError("Invalid input stream type.");
}
}
cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp());
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
template <typename U>
::mediapipe::Status ConcatenateVectors(std::false_type,
CalculatorContext* cc) {
mediapipe::Status ConcatenateVectors(std::false_type, CalculatorContext* cc) {
return ConsumeAndConcatenateVectors<T>(std::is_move_constructible<U>(), cc);
}
template <typename U>
::mediapipe::Status ConsumeAndConcatenateVectors(std::true_type,
CalculatorContext* cc) {
mediapipe::Status ConsumeAndConcatenateVectors(std::true_type,
CalculatorContext* cc) {
auto output = absl::make_unique<std::vector<U>>();
for (int i = 0; i < cc->Inputs().NumEntries(); ++i) {
auto& input = cc->Inputs().Index(i);
@@ -105,7 +103,7 @@ class ConcatenateVectorCalculator : public CalculatorBase {
if (input.IsEmpty()) continue;
if (input.Value().ValidateAsType<U>().ok()) {
::mediapipe::StatusOr<std::unique_ptr<U>> value_status =
mediapipe::StatusOr<std::unique_ptr<U>> value_status =
input.Value().Consume<U>();
if (value_status.ok()) {
std::unique_ptr<U> value = std::move(value_status).ValueOrDie();
@@ -114,7 +112,7 @@ class ConcatenateVectorCalculator : public CalculatorBase {
return value_status.status();
}
} else if (input.Value().ValidateAsType<std::vector<U>>().ok()) {
::mediapipe::StatusOr<std::unique_ptr<std::vector<U>>> value_status =
mediapipe::StatusOr<std::unique_ptr<std::vector<U>>> value_status =
input.Value().Consume<std::vector<U>>();
if (value_status.ok()) {
std::unique_ptr<std::vector<U>> value =
@@ -125,17 +123,17 @@ class ConcatenateVectorCalculator : public CalculatorBase {
return value_status.status();
}
} else {
return ::mediapipe::InvalidArgumentError("Invalid input stream type.");
return mediapipe::InvalidArgumentError("Invalid input stream type.");
}
}
cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp());
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
template <typename U>
::mediapipe::Status ConsumeAndConcatenateVectors(std::false_type,
CalculatorContext* cc) {
return ::mediapipe::InternalError(
mediapipe::Status ConsumeAndConcatenateVectors(std::false_type,
CalculatorContext* cc) {
return mediapipe::InternalError(
"Cannot copy or move inputs to concatenate them");
}
mediapipe/calculators/core/constant_side_packet_calculator.cc
+8 -8
View File
@@ -54,7 +54,7 @@ namespace {} // namespace
// }
class ConstantSidePacketCalculator : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
static mediapipe::Status GetContract(CalculatorContract* cc) {
const auto& options =
cc->Options<::mediapipe::ConstantSidePacketCalculatorOptions>();
RET_CHECK_EQ(cc->OutputSidePackets().NumEntries(kPacketTag),
@@ -80,14 +80,14 @@ class ConstantSidePacketCalculator : public CalculatorBase {
} else if (packet_options.has_classification_list_value()) {
packet.Set<ClassificationList>();
} else {
return ::mediapipe::InvalidArgumentError(
return mediapipe::InvalidArgumentError(
"None of supported values were specified in options.");
}
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status Open(CalculatorContext* cc) override {
mediapipe::Status Open(CalculatorContext* cc) override {
const auto& options =
cc->Options<::mediapipe::ConstantSidePacketCalculatorOptions>();
int index = 0;
@@ -109,15 +109,15 @@ class ConstantSidePacketCalculator : public CalculatorBase {
packet.Set(MakePacket<ClassificationList>(
packet_options.classification_list_value()));
} else {
return ::mediapipe::InvalidArgumentError(
return mediapipe::InvalidArgumentError(
"None of supported values were specified in options.");
}
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status Process(CalculatorContext* cc) override {
return ::mediapipe::OkStatus();
mediapipe::Status Process(CalculatorContext* cc) override {
return mediapipe::OkStatus();
}
private:
mediapipe/calculators/core/constant_side_packet_calculator_test.cc
+5 -5
View File
@@ -40,7 +40,7 @@ void DoTestSingleSidePacket(absl::string_view packet_spec,
}
)";
CalculatorGraphConfig graph_config =
::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
absl::Substitute(graph_config_template, packet_spec));
CalculatorGraph graph;
MP_ASSERT_OK(graph.Initialize(graph_config));
@@ -62,7 +62,7 @@ TEST(ConstantSidePacketCalculatorTest, EveryPossibleType) {
TEST(ConstantSidePacketCalculatorTest, MultiplePackets) {
CalculatorGraphConfig graph_config =
::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
node {
calculator: "ConstantSidePacketCalculator"
output_side_packet: "PACKET:0:int_packet"
@@ -115,7 +115,7 @@ TEST(ConstantSidePacketCalculatorTest, MultiplePackets) {
TEST(ConstantSidePacketCalculatorTest, ProcessingPacketsWithCorrectTagOnly) {
CalculatorGraphConfig graph_config =
::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
node {
calculator: "ConstantSidePacketCalculator"
output_side_packet: "PACKET:0:int_packet"
@@ -159,7 +159,7 @@ TEST(ConstantSidePacketCalculatorTest, ProcessingPacketsWithCorrectTagOnly) {
TEST(ConstantSidePacketCalculatorTest, IncorrectConfig_MoreOptionsThanPackets) {
CalculatorGraphConfig graph_config =
::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
node {
calculator: "ConstantSidePacketCalculator"
output_side_packet: "PACKET:int_packet"
@@ -177,7 +177,7 @@ TEST(ConstantSidePacketCalculatorTest, IncorrectConfig_MoreOptionsThanPackets) {
TEST(ConstantSidePacketCalculatorTest, IncorrectConfig_MorePacketsThanOptions) {
CalculatorGraphConfig graph_config =
::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
node {
calculator: "ConstantSidePacketCalculator"
output_side_packet: "PACKET:0:int_packet"
mediapipe/calculators/core/counting_source_calculator.cc
+8 -8
View File
@@ -30,7 +30,7 @@ namespace mediapipe {
// provided, then batches are of size 1.
class CountingSourceCalculator : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
static mediapipe::Status GetContract(CalculatorContract* cc) {
cc->Outputs().Index(0).Set<int>();
if (cc->InputSidePackets().HasTag("ERROR_ON_OPEN")) {
@@ -55,13 +55,13 @@ class CountingSourceCalculator : public CalculatorBase {
if (cc->InputSidePackets().HasTag("INCREMENT")) {
cc->InputSidePackets().Tag("INCREMENT").Set<int>();
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status Open(CalculatorContext* cc) override {
mediapipe::Status Open(CalculatorContext* cc) override {
if (cc->InputSidePackets().HasTag("ERROR_ON_OPEN") &&
cc->InputSidePackets().Tag("ERROR_ON_OPEN").Get<bool>()) {
return ::mediapipe::NotFoundError("expected error");
return mediapipe::NotFoundError("expected error");
}
if (cc->InputSidePackets().HasTag("ERROR_COUNT")) {
error_count_ = cc->InputSidePackets().Tag("ERROR_COUNT").Get<int>();
@@ -83,12 +83,12 @@ class CountingSourceCalculator : public CalculatorBase {
RET_CHECK_LT(0, increment_);
}
RET_CHECK(error_count_ >= 0 || max_count_ >= 0);
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status Process(CalculatorContext* cc) override {
mediapipe::Status Process(CalculatorContext* cc) override {
if (error_count_ >= 0 && batch_counter_ >= error_count_) {
return ::mediapipe::InternalError("expected error");
return mediapipe::InternalError("expected error");
}
if (max_count_ >= 0 && batch_counter_ >= max_count_) {
return tool::StatusStop();
@@ -98,7 +98,7 @@ class CountingSourceCalculator : public CalculatorBase {
counter_ += increment_;
}
++batch_counter_;
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
private:
mediapipe/calculators/core/dequantize_byte_array_calculator.cc
+8 -8
View File
@@ -37,34 +37,34 @@ namespace mediapipe {
class DequantizeByteArrayCalculator : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
static mediapipe::Status GetContract(CalculatorContract* cc) {
cc->Inputs().Tag("ENCODED").Set<std::string>();
cc->Outputs().Tag("FLOAT_VECTOR").Set<std::vector<float>>();
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status Open(CalculatorContext* cc) final {
mediapipe::Status Open(CalculatorContext* cc) final {
const auto options =
cc->Options<::mediapipe::DequantizeByteArrayCalculatorOptions>();
if (!options.has_max_quantized_value() ||
!options.has_min_quantized_value()) {
return ::mediapipe::InvalidArgumentError(
return mediapipe::InvalidArgumentError(
"Both max_quantized_value and min_quantized_value must be provided "
"in DequantizeByteArrayCalculatorOptions.");
}
float max_quantized_value = options.max_quantized_value();
float min_quantized_value = options.min_quantized_value();
if (max_quantized_value < min_quantized_value + FLT_EPSILON) {
return ::mediapipe::InvalidArgumentError(
return mediapipe::InvalidArgumentError(
"max_quantized_value must be greater than min_quantized_value.");
}
float range = max_quantized_value - min_quantized_value;
scalar_ = range / 255.0;
bias_ = (range / 512.0) + min_quantized_value;
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status Process(CalculatorContext* cc) final {
mediapipe::Status Process(CalculatorContext* cc) final {
const std::string& encoded =
cc->Inputs().Tag("ENCODED").Value().Get<std::string>();
std::vector<float> float_vector;
@@ -77,7 +77,7 @@ class DequantizeByteArrayCalculator : public CalculatorBase {
.Tag("FLOAT_VECTOR")
.AddPacket(MakePacket<std::vector<float>>(float_vector)
.At(cc->InputTimestamp()));
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
private:
mediapipe/calculators/core/end_loop_calculator.h
+4 -4
View File
@@ -57,7 +57,7 @@ class EndLoopCalculator : public CalculatorBase {
using ItemT = typename IterableT::value_type;
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
static mediapipe::Status GetContract(CalculatorContract* cc) {
RET_CHECK(cc->Inputs().HasTag("BATCH_END"))
<< "Missing BATCH_END tagged input_stream.";
cc->Inputs().Tag("BATCH_END").Set<Timestamp>();
@@ -67,10 +67,10 @@ class EndLoopCalculator : public CalculatorBase {
RET_CHECK(cc->Outputs().HasTag("ITERABLE"));
cc->Outputs().Tag("ITERABLE").Set<IterableT>();
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status Process(CalculatorContext* cc) override {
mediapipe::Status Process(CalculatorContext* cc) override {
if (!cc->Inputs().Tag("ITEM").IsEmpty()) {
if (!input_stream_collection_) {
input_stream_collection_.reset(new IterableT);
@@ -94,7 +94,7 @@ class EndLoopCalculator : public CalculatorBase {
.SetNextTimestampBound(Timestamp(loop_control_ts.Value() + 1));
}
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
private:
mediapipe/calculators/core/flow_limiter_calculator.cc
+159 -129
View File
@@ -16,6 +16,7 @@
#include <utility>
#include <vector>
#include "mediapipe/calculators/core/flow_limiter_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status.h"
@@ -23,41 +24,23 @@
namespace mediapipe {
// FlowLimiterCalculator is used to limit the number of pipelined processing
// operations in a section of the graph.
// FlowLimiterCalculator is used to limit the number of frames in flight
// by dropping input frames when necessary.
//
// Typical topology:
// The input stream "FINISH" is used to signal the FlowLimiterCalculator
// when a frame is finished processing. Either a non-empty "FINISH" packet
// or a timestamp bound should be received for each processed frame.
//
// in ->-[FLC]-[foo]-...-[bar]-+->- out
// ^_____________________|
// FINISHED
// The combination of `max_in_flight: 1` and `max_in_queue: 1` generally gives
// best throughput/latency balance. Throughput is nearly optimal as the
// graph is never idle as there is always something in the queue. Latency is
// nearly optimal latency as the queue always stores the latest available frame.
//
// By connecting the output of the graph section to this calculator's FINISHED
// input with a backwards edge, this allows FLC to keep track of how many
// timestamps are currently being processed.
//
// The limit defaults to 1, and can be overridden with the MAX_IN_FLIGHT side
// packet.
//
// As long as the number of timestamps being processed ("in flight") is below
// the limit, FLC allows input to pass through. When the limit is reached,
// FLC starts dropping input packets, keeping only the most recent. When the
// processing count decreases again, as signaled by the receipt of a packet on
// FINISHED, FLC allows packets to flow again, releasing the most recently
// queued packet, if any.
//
// If there are multiple input streams, packet dropping is synchronized.
//
// IMPORTANT: for each timestamp where FLC forwards a packet (or a set of
// packets, if using multiple data streams), a packet must eventually arrive on
// the FINISHED stream. Dropping packets in the section between FLC and
// FINISHED will make the in-flight count incorrect.
//
// TODO: Remove this comment when graph-level ISH has been removed.
// NOTE: this calculator should always use the ImmediateInputStreamHandler and
// uses it by default. However, if the graph specifies a graph-level
// InputStreamHandler, to override that setting, the InputStreamHandler must
// be explicitly specified as shown below.
// Increasing `max_in_flight` to 2 or more can yield the better throughput
// when the graph exhibits a high degree of pipeline parallelism. Decreasing
// `max_in_flight` to 0 can yield a better average latency, but at the cost of
// lower throughput (lower framerate) due to the time during which the graph
// is idle awaiting the next input frame.
//
// Example config:
// node {
@@ -68,131 +51,178 @@ namespace mediapipe {
// tag_index: 'FINISHED'
// back_edge: true
// }
// input_stream_handler {
// input_stream_handler: 'ImmediateInputStreamHandler'
// }
// output_stream: "gated_frames"
// output_stream: "sampled_frames"
// output_stream: "ALLOW:allowed_timestamps"
// }
//
// The "ALLOW" stream indicates the transition between accepting frames and
// dropping frames. "ALLOW = true" indicates the start of accepting frames
// including the current timestamp, and "ALLOW = true" indicates the start of
// dropping frames including the current timestamp.
//
// FlowLimiterCalculator provides limited support for multiple input streams.
// The first input stream is treated as the main input stream and successive
// input streams are treated as auxiliary input streams. The auxiliary input
// streams are limited to timestamps passed on the main input stream.
//
class FlowLimiterCalculator : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
int num_data_streams = cc->Inputs().NumEntries("");
RET_CHECK_GE(num_data_streams, 1);
RET_CHECK_EQ(cc->Outputs().NumEntries(""), num_data_streams)
<< "Output streams must correspond input streams except for the "
"finish indicator input stream.";
for (int i = 0; i < num_data_streams; ++i) {
static mediapipe::Status GetContract(CalculatorContract* cc) {
auto& side_inputs = cc->InputSidePackets();
side_inputs.Tag("OPTIONS").Set<FlowLimiterCalculatorOptions>().Optional();
cc->Inputs().Tag("OPTIONS").Set<FlowLimiterCalculatorOptions>().Optional();
RET_CHECK_GE(cc->Inputs().NumEntries(""), 1);
for (int i = 0; i < cc->Inputs().NumEntries(""); ++i) {
cc->Inputs().Get("", i).SetAny();
cc->Outputs().Get("", i).SetSameAs(&(cc->Inputs().Get("", i)));
}
cc->Inputs().Get("FINISHED", 0).SetAny();
if (cc->InputSidePackets().HasTag("MAX_IN_FLIGHT")) {
cc->InputSidePackets().Tag("MAX_IN_FLIGHT").Set<int>();
}
if (cc->Outputs().HasTag("ALLOW")) {
cc->Outputs().Tag("ALLOW").Set<bool>();
}
cc->InputSidePackets().Tag("MAX_IN_FLIGHT").Set<int>().Optional();
cc->Outputs().Tag("ALLOW").Set<bool>().Optional();
cc->SetInputStreamHandler("ImmediateInputStreamHandler");
return ::mediapipe::OkStatus();
cc->SetProcessTimestampBounds(true);
return mediapipe::OkStatus();
}
::mediapipe::Status Open(CalculatorContext* cc) final {
finished_id_ = cc->Inputs().GetId("FINISHED", 0);
max_in_flight_ = 1;
mediapipe::Status Open(CalculatorContext* cc) final {
options_ = cc->Options<FlowLimiterCalculatorOptions>();
options_ = tool::RetrieveOptions(options_, cc->InputSidePackets());
if (cc->InputSidePackets().HasTag("MAX_IN_FLIGHT")) {
max_in_flight_ = cc->InputSidePackets().Tag("MAX_IN_FLIGHT").Get<int>();
options_.set_max_in_flight(
cc->InputSidePackets().Tag("MAX_IN_FLIGHT").Get<int>());
}
RET_CHECK_GE(max_in_flight_, 1);
num_in_flight_ = 0;
allowed_id_ = cc->Outputs().GetId("ALLOW", 0);
allow_ctr_ts_ = Timestamp(0);
num_data_streams_ = cc->Inputs().NumEntries("");
data_stream_bound_ts_.resize(num_data_streams_);
input_queues_.resize(cc->Inputs().NumEntries(""));
RET_CHECK_OK(CopyInputHeadersToOutputs(cc->Inputs(), &(cc->Outputs())));
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
bool Allow() { return num_in_flight_ < max_in_flight_; }
// Returns true if an additional frame can be released for processing.
// The "ALLOW" output stream indicates this condition at each input frame.
bool ProcessingAllowed() {
return frames_in_flight_.size() < options_.max_in_flight();
}
::mediapipe::Status Process(CalculatorContext* cc) final {
bool old_allow = Allow();
Timestamp lowest_incomplete_ts = Timestamp::Done();
// Process FINISHED stream.
if (!cc->Inputs().Get(finished_id_).Value().IsEmpty()) {
RET_CHECK_GT(num_in_flight_, 0)
<< "Received a FINISHED packet, but we had none in flight.";
--num_in_flight_;
// Outputs a packet indicating whether a frame was sent or dropped.
void SendAllow(bool allow, Timestamp ts, CalculatorContext* cc) {
if (cc->Outputs().HasTag("ALLOW")) {
cc->Outputs().Tag("ALLOW").AddPacket(MakePacket<bool>(allow).At(ts));
}
}
// Process data streams.
for (int i = 0; i < num_data_streams_; ++i) {
auto& stream = cc->Inputs().Get("", i);
auto& out = cc->Outputs().Get("", i);
Packet& packet = stream.Value();
auto ts = packet.Timestamp();
if (ts.IsRangeValue() && data_stream_bound_ts_[i] <= ts) {
data_stream_bound_ts_[i] = ts + 1;
// Note: it's ok to update the output bound here, before sending the
// packet, because updates are batched during the Process function.
out.SetNextTimestampBound(data_stream_bound_ts_[i]);
}
lowest_incomplete_ts =
std::min(lowest_incomplete_ts, data_stream_bound_ts_[i]);
// Sets the timestamp bound or closes an output stream.
void SetNextTimestampBound(Timestamp bound, OutputStream* stream) {
if (bound > Timestamp::Max()) {
stream->Close();
} else {
stream->SetNextTimestampBound(bound);
}
}
if (packet.IsEmpty()) {
// If the input stream is closed, close the corresponding output.
if (stream.IsDone() && !out.IsClosed()) {
out.Close();
// Returns true if a certain timestamp is being processed.
bool IsInFlight(Timestamp timestamp) {
return std::find(frames_in_flight_.begin(), frames_in_flight_.end(),
timestamp) != frames_in_flight_.end();
}
// Releases input packets up to the latest settled input timestamp.
void ProcessAuxiliaryInputs(CalculatorContext* cc) {
Timestamp settled_bound = cc->Outputs().Get("", 0).NextTimestampBound();
for (int i = 1; i < cc->Inputs().NumEntries(""); ++i) {
// Release settled frames from each input queue.
while (!input_queues_[i].empty() &&
input_queues_[i].front().Timestamp() < settled_bound) {
Packet packet = input_queues_[i].front();
input_queues_[i].pop_front();
if (IsInFlight(packet.Timestamp())) {
cc->Outputs().Get("", i).AddPacket(packet);
}
// TODO: if the packet is empty, the ts is unset, and we
// cannot read the timestamp bound, even though we'd like to propagate
// it.
} else if (mediapipe::ContainsKey(pending_ts_, ts)) {
// If we have already sent this timestamp (on another stream), send it
// on this stream too.
out.AddPacket(std::move(packet));
} else if (Allow() && (ts > last_dropped_ts_)) {
// If the in-flight is under the limit, and if we have not already
// dropped this or a later timestamp on another stream, then send
// the packet and add an in-flight timestamp.
out.AddPacket(std::move(packet));
pending_ts_.insert(ts);
++num_in_flight_;
}
// Propagate each input timestamp bound.
if (!input_queues_[i].empty()) {
Timestamp bound = input_queues_[i].front().Timestamp();
SetNextTimestampBound(bound, &cc->Outputs().Get("", i));
} else {
// Otherwise, we'll drop the packet.
last_dropped_ts_ = std::max(last_dropped_ts_, ts);
Timestamp bound =
cc->Inputs().Get("", i).Value().Timestamp().NextAllowedInStream();
SetNextTimestampBound(bound, &cc->Outputs().Get("", i));
}
}
}
// Releases input packets allowed by the max_in_flight constraint.
mediapipe::Status Process(CalculatorContext* cc) final {
options_ = tool::RetrieveOptions(options_, cc->Inputs());
// Process the FINISHED input stream.
Packet finished_packet = cc->Inputs().Tag("FINISHED").Value();
if (finished_packet.Timestamp() == cc->InputTimestamp()) {
while (!frames_in_flight_.empty() &&
frames_in_flight_.front() <= finished_packet.Timestamp()) {
frames_in_flight_.pop_front();
}
}
// Remove old pending_ts_ entries.
auto it = std::lower_bound(pending_ts_.begin(), pending_ts_.end(),
lowest_incomplete_ts);
pending_ts_.erase(pending_ts_.begin(), it);
// Update ALLOW signal.
if ((old_allow != Allow()) && allowed_id_.IsValid()) {
cc->Outputs()
.Get(allowed_id_)
.AddPacket(MakePacket<bool>(Allow()).At(++allow_ctr_ts_));
// Process the frame input streams.
for (int i = 0; i < cc->Inputs().NumEntries(""); ++i) {
Packet packet = cc->Inputs().Get("", i).Value();
if (!packet.IsEmpty()) {
input_queues_[i].push_back(packet);
}
}
return ::mediapipe::OkStatus();
// Abandon expired frames in flight. Note that old frames are abandoned
// when much newer frame timestamps arrive regardless of elapsed time.
TimestampDiff timeout = options_.in_flight_timeout();
Timestamp latest_ts = cc->Inputs().Get("", 0).Value().Timestamp();
if (timeout > 0 && latest_ts == cc->InputTimestamp() &&
latest_ts < Timestamp::Max()) {
while (!frames_in_flight_.empty() &&
(latest_ts - frames_in_flight_.front()) > timeout) {
frames_in_flight_.pop_front();
}
}
// Release allowed frames from the main input queue.
auto& input_queue = input_queues_[0];
while (ProcessingAllowed() && !input_queue.empty()) {
Packet packet = input_queue.front();
input_queue.pop_front();
cc->Outputs().Get("", 0).AddPacket(packet);
SendAllow(true, packet.Timestamp(), cc);
frames_in_flight_.push_back(packet.Timestamp());
}
// Limit the number of queued frames.
// Note that frames can be dropped after frames are released because
// frame-packets and FINISH-packets never arrive in the same Process call.
while (input_queue.size() > options_.max_in_queue()) {
Packet packet = input_queue.front();
input_queue.pop_front();
SendAllow(false, packet.Timestamp(), cc);
}
// Propagate the input timestamp bound.
if (!input_queue.empty()) {
Timestamp bound = input_queue.front().Timestamp();
SetNextTimestampBound(bound, &cc->Outputs().Get("", 0));
} else {
Timestamp bound =
cc->Inputs().Get("", 0).Value().Timestamp().NextAllowedInStream();
SetNextTimestampBound(bound, &cc->Outputs().Get("", 0));
if (cc->Outputs().HasTag("ALLOW")) {
SetNextTimestampBound(bound, &cc->Outputs().Tag("ALLOW"));
}
}
ProcessAuxiliaryInputs(cc);
return mediapipe::OkStatus();
}
private:
std::set<Timestamp> pending_ts_;
Timestamp last_dropped_ts_;
int num_data_streams_;
int num_in_flight_;
int max_in_flight_;
CollectionItemId finished_id_;
CollectionItemId allowed_id_;
Timestamp allow_ctr_ts_;
std::vector<Timestamp> data_stream_bound_ts_;
FlowLimiterCalculatorOptions options_;
std::vector<std::deque<Packet>> input_queues_;
std::deque<Timestamp> frames_in_flight_;
};
REGISTER_CALCULATOR(FlowLimiterCalculator);
mediapipe/calculators/core/flow_limiter_calculator.proto
+40
View File
@@ -0,0 +1,40 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto2";
package mediapipe;
import "mediapipe/framework/calculator.proto";
option objc_class_prefix = "MediaPipe";
message FlowLimiterCalculatorOptions {
extend mediapipe.CalculatorOptions {
optional FlowLimiterCalculatorOptions ext = 326963320;
}
// The maximum number of frames released for processing at one time.
// The default value limits to 1 frame processing at a time.
optional int32 max_in_flight = 1 [default = 1];
// The maximum number of frames queued waiting for processing.
// The default value limits to 1 frame awaiting processing.
optional int32 max_in_queue = 2 [default = 0];
// The maximum time in microseconds to wait for a frame to finish processing.
// The default value stops waiting after 1 sec.
// The value 0 specifies no timeout.
optional int64 in_flight_timeout = 3 [default = 1000000];
}
mediapipe/calculators/core/flow_limiter_calculator_test.cc
+570 -304
View File
File diff suppressed because it is too large Load Diff
mediapipe/calculators/core/gate_calculator.cc
+9 -9
View File
@@ -82,7 +82,7 @@ class GateCalculator : public CalculatorBase {
public:
GateCalculator() {}
static ::mediapipe::Status CheckAndInitAllowDisallowInputs(
static mediapipe::Status CheckAndInitAllowDisallowInputs(
CalculatorContract* cc) {
bool input_via_side_packet = cc->InputSidePackets().HasTag("ALLOW") ||
cc->InputSidePackets().HasTag("DISALLOW");
@@ -110,10 +110,10 @@ class GateCalculator : public CalculatorBase {
cc->Inputs().Tag("DISALLOW").Set<bool>();
}
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
static mediapipe::Status GetContract(CalculatorContract* cc) {
RET_CHECK_OK(CheckAndInitAllowDisallowInputs(cc));
const int num_data_streams = cc->Inputs().NumEntries("");
@@ -130,10 +130,10 @@ class GateCalculator : public CalculatorBase {
cc->Outputs().Tag("STATE_CHANGE").Set<bool>();
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status Open(CalculatorContext* cc) final {
mediapipe::Status Open(CalculatorContext* cc) final {
use_side_packet_for_allow_disallow_ = false;
if (cc->InputSidePackets().HasTag("ALLOW")) {
use_side_packet_for_allow_disallow_ = true;
@@ -153,10 +153,10 @@ class GateCalculator : public CalculatorBase {
const auto& options = cc->Options<::mediapipe::GateCalculatorOptions>();
empty_packets_as_allow_ = options.empty_packets_as_allow();
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status Process(CalculatorContext* cc) final {
mediapipe::Status Process(CalculatorContext* cc) final {
bool allow = empty_packets_as_allow_;
if (use_side_packet_for_allow_disallow_) {
allow = allow_by_side_packet_decision_;
@@ -195,7 +195,7 @@ class GateCalculator : public CalculatorBase {
cc->Outputs().Get("", i).Close();
}
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
// Process data streams.
@@ -205,7 +205,7 @@ class GateCalculator : public CalculatorBase {
}
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
private:
mediapipe/calculators/core/gate_calculator_test.cc
+1 -1
View File
@@ -25,7 +25,7 @@ namespace {
class GateCalculatorTest : public ::testing::Test {
protected:
// Helper to run a graph and return status.
static ::mediapipe::Status RunGraph(const std::string& proto) {
static mediapipe::Status RunGraph(const std::string& proto) {
auto runner = absl::make_unique<CalculatorRunner>(
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(proto));
return runner->Run();
mediapipe/calculators/core/immediate_mux_calculator.cc
+10 -13
View File
@@ -29,9 +29,7 @@ namespace mediapipe {
// received.
//
// This Calculator can be used with an ImmediateInputStreamHandler or with the
// default ISH. Note that currently ImmediateInputStreamHandler seems to
// interfere with timestamp bound propagation, so it is better to use the
// default unless the immediate one is needed. (b/118387598)
// default ISH.
//
// This Calculator is designed to work with a Demux calculator such as
// the RoundRobinDemuxCalculator. Therefore, packets from different
@@ -45,17 +43,16 @@ class ImmediateMuxCalculator : public CalculatorBase {
public:
// This calculator combines any set of input streams into a single
// output stream. All input stream types must match the output stream type.
static ::mediapipe::Status GetContract(CalculatorContract* cc);
static mediapipe::Status GetContract(CalculatorContract* cc);
// Passes any input packet to the output stream immediately, unless the
// packet timestamp is lower than a previously passed packet.
::mediapipe::Status Process(CalculatorContext* cc) override;
::mediapipe::Status Open(CalculatorContext* cc) override;
mediapipe::Status Process(CalculatorContext* cc) override;
mediapipe::Status Open(CalculatorContext* cc) override;
};
REGISTER_CALCULATOR(ImmediateMuxCalculator);
::mediapipe::Status ImmediateMuxCalculator::GetContract(
CalculatorContract* cc) {
mediapipe::Status ImmediateMuxCalculator::GetContract(CalculatorContract* cc) {
RET_CHECK(cc->Outputs().NumEntries() >= 1 && cc->Outputs().NumEntries() <= 2)
<< "This calculator produces only one or two output streams.";
cc->Outputs().Index(0).SetAny();
@@ -65,15 +62,15 @@ REGISTER_CALCULATOR(ImmediateMuxCalculator);
for (int i = 0; i < cc->Inputs().NumEntries(); ++i) {
cc->Inputs().Index(i).SetSameAs(&cc->Outputs().Index(0));
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status ImmediateMuxCalculator::Open(CalculatorContext* cc) {
mediapipe::Status ImmediateMuxCalculator::Open(CalculatorContext* cc) {
cc->SetOffset(TimestampDiff(0));
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status ImmediateMuxCalculator::Process(CalculatorContext* cc) {
mediapipe::Status ImmediateMuxCalculator::Process(CalculatorContext* cc) {
// Pass along the first packet, unless it has been superseded.
for (int i = 0; i < cc->Inputs().NumEntries(); ++i) {
const Packet& packet = cc->Inputs().Index(i).Value();
@@ -91,7 +88,7 @@ REGISTER_CALCULATOR(ImmediateMuxCalculator);
}
}
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
} // namespace mediapipe
mediapipe/calculators/core/immediate_mux_calculator_test.cc
+6 -6
View File
@@ -289,19 +289,19 @@ TEST_F(ImmediateMuxCalculatorTest, SimultaneousTimestamps) {
}
// A Calculator::Process callback function.
typedef std::function<::mediapipe::Status(const InputStreamShardSet&,
OutputStreamShardSet*)>
typedef std::function<mediapipe::Status(const InputStreamShardSet&,
OutputStreamShardSet*)>
ProcessFunction;
// A testing callback function that passes through all packets.
::mediapipe::Status PassThrough(const InputStreamShardSet& inputs,
OutputStreamShardSet* outputs) {
mediapipe::Status PassThrough(const InputStreamShardSet& inputs,
OutputStreamShardSet* outputs) {
for (int i = 0; i < inputs.NumEntries(); ++i) {
if (!inputs.Index(i).Value().IsEmpty()) {
outputs->Index(i).AddPacket(inputs.Index(i).Value());
}
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
TEST_F(ImmediateMuxCalculatorTest, Demux) {
@@ -325,7 +325,7 @@ TEST_F(ImmediateMuxCalculatorTest, Demux) {
auto out_cb = [&](const Packet& p) {
absl::MutexLock lock(&out_mutex);
out_packets.push_back(p);
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
};
auto wait_for = [&](std::function<bool()> cond) {
absl::MutexLock lock(&out_mutex);
mediapipe/calculators/core/make_pair_calculator.cc
+6 -6
View File
@@ -35,24 +35,24 @@ class MakePairCalculator : public CalculatorBase {
MakePairCalculator() {}
~MakePairCalculator() override {}
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
static mediapipe::Status GetContract(CalculatorContract* cc) {
cc->Inputs().Index(0).SetAny();
cc->Inputs().Index(1).SetAny();
cc->Outputs().Index(0).Set<std::pair<Packet, Packet>>();
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status Open(CalculatorContext* cc) override {
mediapipe::Status Open(CalculatorContext* cc) override {
cc->SetOffset(TimestampDiff(0));
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status Process(CalculatorContext* cc) override {
mediapipe::Status Process(CalculatorContext* cc) override {
cc->Outputs().Index(0).Add(
new std::pair<Packet, Packet>(cc->Inputs().Index(0).Value(),
cc->Inputs().Index(1).Value()),
cc->InputTimestamp());
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
};
mediapipe/calculators/core/matrix_multiply_calculator.cc
+9 -9
View File
@@ -33,34 +33,34 @@ class MatrixMultiplyCalculator : public CalculatorBase {
MatrixMultiplyCalculator() {}
~MatrixMultiplyCalculator() override {}
static ::mediapipe::Status GetContract(CalculatorContract* cc);
static mediapipe::Status GetContract(CalculatorContract* cc);
::mediapipe::Status Open(CalculatorContext* cc) override;
::mediapipe::Status Process(CalculatorContext* cc) override;
mediapipe::Status Open(CalculatorContext* cc) override;
mediapipe::Status Process(CalculatorContext* cc) override;
};
REGISTER_CALCULATOR(MatrixMultiplyCalculator);
// static
::mediapipe::Status MatrixMultiplyCalculator::GetContract(
mediapipe::Status MatrixMultiplyCalculator::GetContract(
CalculatorContract* cc) {
cc->Inputs().Index(0).Set<Matrix>();
cc->Outputs().Index(0).Set<Matrix>();
cc->InputSidePackets().Index(0).Set<Matrix>();
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status MatrixMultiplyCalculator::Open(CalculatorContext* cc) {
mediapipe::Status MatrixMultiplyCalculator::Open(CalculatorContext* cc) {
// The output is at the same timestamp as the input.
cc->SetOffset(TimestampDiff(0));
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status MatrixMultiplyCalculator::Process(CalculatorContext* cc) {
mediapipe::Status MatrixMultiplyCalculator::Process(CalculatorContext* cc) {
Matrix* multiplied = new Matrix();
*multiplied = cc->InputSidePackets().Index(0).Get<Matrix>() *
cc->Inputs().Index(0).Get<Matrix>();
cc->Outputs().Index(0).Add(multiplied, cc->InputTimestamp());
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
} // namespace mediapipe
mediapipe/calculators/core/matrix_subtract_calculator.cc
+13 -13
View File
@@ -46,10 +46,10 @@ class MatrixSubtractCalculator : public CalculatorBase {
MatrixSubtractCalculator() {}
~MatrixSubtractCalculator() override {}
static ::mediapipe::Status GetContract(CalculatorContract* cc);
static mediapipe::Status GetContract(CalculatorContract* cc);
::mediapipe::Status Open(CalculatorContext* cc) override;
::mediapipe::Status Process(CalculatorContext* cc) override;
mediapipe::Status Open(CalculatorContext* cc) override;
mediapipe::Status Process(CalculatorContext* cc) override;
private:
bool subtract_from_input_ = false;
@@ -57,11 +57,11 @@ class MatrixSubtractCalculator : public CalculatorBase {
REGISTER_CALCULATOR(MatrixSubtractCalculator);
// static
::mediapipe::Status MatrixSubtractCalculator::GetContract(
mediapipe::Status MatrixSubtractCalculator::GetContract(
CalculatorContract* cc) {
if (cc->Inputs().NumEntries() != 1 ||
cc->InputSidePackets().NumEntries() != 1) {
return ::mediapipe::InvalidArgumentError(
return mediapipe::InvalidArgumentError(
"MatrixSubtractCalculator only accepts exactly one input stream and "
"one "
"input side packet");
@@ -75,23 +75,23 @@ REGISTER_CALCULATOR(MatrixSubtractCalculator);
cc->Inputs().Tag("SUBTRAHEND").Set<Matrix>();
cc->InputSidePackets().Tag("MINUEND").Set<Matrix>();
} else {
return ::mediapipe::InvalidArgumentError(
return mediapipe::InvalidArgumentError(
"Must specify exactly one minuend and one subtrahend.");
}
cc->Outputs().Index(0).Set<Matrix>();
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status MatrixSubtractCalculator::Open(CalculatorContext* cc) {
mediapipe::Status MatrixSubtractCalculator::Open(CalculatorContext* cc) {
// The output is at the same timestamp as the input.
cc->SetOffset(TimestampDiff(0));
if (cc->Inputs().HasTag("MINUEND")) {
subtract_from_input_ = true;
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status MatrixSubtractCalculator::Process(CalculatorContext* cc) {
mediapipe::Status MatrixSubtractCalculator::Process(CalculatorContext* cc) {
Matrix* subtracted = new Matrix();
if (subtract_from_input_) {
const Matrix& input_matrix = cc->Inputs().Tag("MINUEND").Get<Matrix>();
@@ -99,7 +99,7 @@ REGISTER_CALCULATOR(MatrixSubtractCalculator);
cc->InputSidePackets().Tag("SUBTRAHEND").Get<Matrix>();
if (input_matrix.rows() != side_input_matrix.rows() ||
input_matrix.cols() != side_input_matrix.cols()) {
return ::mediapipe::InvalidArgumentError(
return mediapipe::InvalidArgumentError(
"Input matrix and the input side matrix must have the same "
"dimension.");
}
@@ -110,14 +110,14 @@ REGISTER_CALCULATOR(MatrixSubtractCalculator);
cc->InputSidePackets().Tag("MINUEND").Get<Matrix>();
if (input_matrix.rows() != side_input_matrix.rows() ||
input_matrix.cols() != side_input_matrix.cols()) {
return ::mediapipe::InvalidArgumentError(
return mediapipe::InvalidArgumentError(
"Input matrix and the input side matrix must have the same "
"dimension.");
}
*subtracted = side_input_matrix - input_matrix;
}
cc->Outputs().Index(0).Add(subtracted, cc->InputTimestamp());
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
} // namespace mediapipe
mediapipe/calculators/core/matrix_to_vector_calculator.cc
+8 -8
View File
@@ -42,30 +42,30 @@ namespace mediapipe {
// }
class MatrixToVectorCalculator : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
static mediapipe::Status GetContract(CalculatorContract* cc) {
cc->Inputs().Index(0).Set<Matrix>(
// Input Packet containing a Matrix.
);
cc->Outputs().Index(0).Set<std::vector<float>>(
// Output Packet containing a vector, one for each input Packet.
);
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status Open(CalculatorContext* cc) override;
mediapipe::Status Open(CalculatorContext* cc) override;
// Outputs a packet containing a vector for each input packet.
::mediapipe::Status Process(CalculatorContext* cc) override;
mediapipe::Status Process(CalculatorContext* cc) override;
};
REGISTER_CALCULATOR(MatrixToVectorCalculator);
::mediapipe::Status MatrixToVectorCalculator::Open(CalculatorContext* cc) {
mediapipe::Status MatrixToVectorCalculator::Open(CalculatorContext* cc) {
// Inform the framework that we don't alter timestamps.
cc->SetOffset(mediapipe::TimestampDiff(0));
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status MatrixToVectorCalculator::Process(CalculatorContext* cc) {
mediapipe::Status MatrixToVectorCalculator::Process(CalculatorContext* cc) {
const Matrix& input = cc->Inputs().Index(0).Get<Matrix>();
auto output = absl::make_unique<std::vector<float>>();
@@ -77,7 +77,7 @@ REGISTER_CALCULATOR(MatrixToVectorCalculator);
output_as_matrix = input;
cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp());
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
} // namespace mediapipe
mediapipe/calculators/core/merge_calculator.cc
+7 -7
View File
@@ -43,7 +43,7 @@ namespace mediapipe {
//
class MergeCalculator : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
static mediapipe::Status GetContract(CalculatorContract* cc) {
RET_CHECK_GT(cc->Inputs().NumEntries(), 0)
<< "Needs at least one input stream";
RET_CHECK_EQ(cc->Outputs().NumEntries(), 1);
@@ -60,29 +60,29 @@ class MergeCalculator : public CalculatorBase {
}
cc->Outputs().Index(0).SetAny();
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status Open(CalculatorContext* cc) final {
mediapipe::Status Open(CalculatorContext* cc) final {
cc->SetOffset(TimestampDiff(0));
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status Process(CalculatorContext* cc) final {
mediapipe::Status Process(CalculatorContext* cc) final {
// Output the packet from the first input stream with a packet ready at this
// timestamp.
for (int i = 0; i < cc->Inputs().NumEntries(); ++i) {
if (!cc->Inputs().Index(i).IsEmpty()) {
cc->Outputs().Index(0).AddPacket(cc->Inputs().Index(i).Value());
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
}
LOG(WARNING) << "Empty input packets at timestamp "
<< cc->InputTimestamp().Value();
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
};
mediapipe/calculators/core/mux_calculator.cc
+8 -8
View File
@@ -36,7 +36,7 @@ constexpr char kInputTag[] = "INPUT";
// with DefaultInputStreamHandler.
class MuxCalculator : public CalculatorBase {
public:
static ::mediapipe::Status CheckAndInitAllowDisallowInputs(
static mediapipe::Status CheckAndInitAllowDisallowInputs(
CalculatorContract* cc) {
RET_CHECK(cc->Inputs().HasTag(kSelectTag) ^
cc->InputSidePackets().HasTag(kSelectTag));
@@ -45,10 +45,10 @@ class MuxCalculator : public CalculatorBase {
} else {
cc->InputSidePackets().Tag(kSelectTag).Set<int>();
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
static mediapipe::Status GetContract(CalculatorContract* cc) {
RET_CHECK_OK(CheckAndInitAllowDisallowInputs(cc));
CollectionItemId data_input_id = cc->Inputs().BeginId(kInputTag);
PacketType* data_input0 = &cc->Inputs().Get(data_input_id);
@@ -64,10 +64,10 @@ class MuxCalculator : public CalculatorBase {
MediaPipeOptions options;
cc->SetInputStreamHandlerOptions(options);
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status Open(CalculatorContext* cc) final {
mediapipe::Status Open(CalculatorContext* cc) final {
use_side_packet_select_ = false;
if (cc->InputSidePackets().HasTag(kSelectTag)) {
use_side_packet_select_ = true;
@@ -79,10 +79,10 @@ class MuxCalculator : public CalculatorBase {
num_data_inputs_ = cc->Inputs().NumEntries(kInputTag);
output_ = cc->Outputs().GetId("OUTPUT", 0);
cc->SetOffset(TimestampDiff(0));
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status Process(CalculatorContext* cc) final {
mediapipe::Status Process(CalculatorContext* cc) final {
int select = use_side_packet_select_
? selected_index_
: cc->Inputs().Get(select_input_).Get<int>();
@@ -91,7 +91,7 @@ class MuxCalculator : public CalculatorBase {
cc->Outputs().Get(output_).AddPacket(
cc->Inputs().Get(data_input_base_ + select).Value());
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
private:
mediapipe/calculators/core/mux_calculator_test.cc
+10 -11
View File
@@ -134,10 +134,9 @@ void RunGraph(const std::string& graph_config_proto,
const std::string& input_stream_name, int num_input_packets,
std::function<Packet(int)> input_fn,
const std::string& output_stream_name,
std::function<::mediapipe::Status(const Packet&)> output_fn) {
std::function<mediapipe::Status(const Packet&)> output_fn) {
CalculatorGraphConfig config =
::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
graph_config_proto);
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(graph_config_proto);
CalculatorGraph graph;
MP_ASSERT_OK(graph.Initialize(config));
MP_ASSERT_OK(graph.ObserveOutputStream(output_stream_name, output_fn));
@@ -166,9 +165,9 @@ TEST(MuxCalculatorTest, InputStreamSelector_DefaultInputStreamHandler) {
// Output and handling.
std::vector<int> output;
// This function collects the output from the packet.
auto output_fn = [&output](const Packet& p) -> ::mediapipe::Status {
auto output_fn = [&output](const Packet& p) -> mediapipe::Status {
output.push_back(p.Get<int>());
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
};
RunGraph(kTestGraphConfig1, {}, kInputName, input_packets.size(), input_fn,
@@ -192,9 +191,9 @@ TEST(MuxCalculatorTest, InputSidePacketSelector_DefaultInputStreamHandler) {
// Output and handling.
std::vector<int> output;
// This function collects the output from the packet.
auto output_fn = [&output](const Packet& p) -> ::mediapipe::Status {
auto output_fn = [&output](const Packet& p) -> mediapipe::Status {
output.push_back(p.Get<int>());
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
};
RunGraph(kTestGraphConfig2, {{kInputSelector, MakePacket<int>(0)}},
@@ -226,9 +225,9 @@ TEST(MuxCalculatorTest, InputStreamSelector_MuxInputStreamHandler) {
// Output and handling.
std::vector<int> output;
// This function collects the output from the packet.
auto output_fn = [&output](const Packet& p) -> ::mediapipe::Status {
auto output_fn = [&output](const Packet& p) -> mediapipe::Status {
output.push_back(p.Get<int>());
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
};
RunGraph(kTestGraphConfig3, {}, kInputName, input_packets.size(), input_fn,
@@ -252,7 +251,7 @@ constexpr char kDualInputGraphConfig[] = R"proto(
TEST(MuxCalculatorTest, DiscardSkippedInputs_MuxInputStreamHandler) {
CalculatorGraphConfig config =
::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
kDualInputGraphConfig);
CalculatorGraph graph;
MP_ASSERT_OK(graph.Initialize(config));
@@ -261,7 +260,7 @@ TEST(MuxCalculatorTest, DiscardSkippedInputs_MuxInputStreamHandler) {
MP_ASSERT_OK(
graph.ObserveOutputStream("test_output", [&output](const Packet& p) {
output = p.Get<std::shared_ptr<int>>();
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}));
MP_ASSERT_OK(graph.StartRun({}));
mediapipe/calculators/core/packet_cloner_calculator.cc
+7 -7
View File
@@ -45,17 +45,17 @@ namespace mediapipe {
// packet_inner_join_calculator.cc: Don't output unless all inputs are new.
class PacketClonerCalculator : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
static mediapipe::Status GetContract(CalculatorContract* cc) {
const int tick_signal_index = cc->Inputs().NumEntries() - 1;
for (int i = 0; i < tick_signal_index; ++i) {
cc->Inputs().Index(i).SetAny();
cc->Outputs().Index(i).SetSameAs(&cc->Inputs().Index(i));
}
cc->Inputs().Index(tick_signal_index).SetAny();
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status Open(CalculatorContext* cc) final {
mediapipe::Status Open(CalculatorContext* cc) final {
// Load options.
const auto calculator_options =
cc->Options<mediapipe::PacketClonerCalculatorOptions>();
@@ -71,10 +71,10 @@ class PacketClonerCalculator : public CalculatorBase {
cc->Outputs().Index(i).SetHeader(cc->Inputs().Index(i).Header());
}
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status Process(CalculatorContext* cc) final {
mediapipe::Status Process(CalculatorContext* cc) final {
// Store input signals.
for (int i = 0; i < tick_signal_index_; ++i) {
if (!cc->Inputs().Index(i).Value().IsEmpty()) {
@@ -88,7 +88,7 @@ class PacketClonerCalculator : public CalculatorBase {
// Return if one of the input is null.
for (int i = 0; i < tick_signal_index_; ++i) {
if (current_[i].IsEmpty()) {
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
}
}
@@ -103,7 +103,7 @@ class PacketClonerCalculator : public CalculatorBase {
}
}
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
private:
mediapipe/calculators/core/packet_inner_join_calculator.cc
+9 -9
View File
@@ -34,10 +34,10 @@ namespace mediapipe {
// packet_cloner_calculator.cc: Repeats last-seen packets from empty inputs.
class PacketInnerJoinCalculator : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc);
static mediapipe::Status GetContract(CalculatorContract* cc);
::mediapipe::Status Open(CalculatorContext* cc) override;
::mediapipe::Status Process(CalculatorContext* cc) override;
mediapipe::Status Open(CalculatorContext* cc) override;
mediapipe::Status Process(CalculatorContext* cc) override;
private:
int num_streams_;
@@ -45,7 +45,7 @@ class PacketInnerJoinCalculator : public CalculatorBase {
REGISTER_CALCULATOR(PacketInnerJoinCalculator);
::mediapipe::Status PacketInnerJoinCalculator::GetContract(
mediapipe::Status PacketInnerJoinCalculator::GetContract(
CalculatorContract* cc) {
RET_CHECK(cc->Inputs().NumEntries() == cc->Outputs().NumEntries())
<< "The number of input and output streams must match.";
@@ -54,25 +54,25 @@ REGISTER_CALCULATOR(PacketInnerJoinCalculator);
cc->Inputs().Index(i).SetAny();
cc->Outputs().Index(i).SetSameAs(&cc->Inputs().Index(i));
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status PacketInnerJoinCalculator::Open(CalculatorContext* cc) {
mediapipe::Status PacketInnerJoinCalculator::Open(CalculatorContext* cc) {
num_streams_ = cc->Inputs().NumEntries();
cc->SetOffset(TimestampDiff(0));
return mediapipe::OkStatus();
}
::mediapipe::Status PacketInnerJoinCalculator::Process(CalculatorContext* cc) {
mediapipe::Status PacketInnerJoinCalculator::Process(CalculatorContext* cc) {
for (int i = 0; i < num_streams_; ++i) {
if (cc->Inputs().Index(i).Value().IsEmpty()) {
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
}
for (int i = 0; i < num_streams_; ++i) {
cc->Outputs().Index(i).AddPacket(cc->Inputs().Index(i).Value());
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
} // namespace mediapipe
mediapipe/calculators/core/packet_presence_calculator.cc
+6 -6
View File
@@ -57,26 +57,26 @@ namespace mediapipe {
// }
class PacketPresenceCalculator : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
static mediapipe::Status GetContract(CalculatorContract* cc) {
cc->Inputs().Tag("PACKET").SetAny();
cc->Outputs().Tag("PRESENCE").Set<bool>();
// Process() function is invoked in response to input stream timestamp
// bound updates.
cc->SetProcessTimestampBounds(true);
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status Open(CalculatorContext* cc) override {
mediapipe::Status Open(CalculatorContext* cc) override {
cc->SetOffset(TimestampDiff(0));
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status Process(CalculatorContext* cc) final {
mediapipe::Status Process(CalculatorContext* cc) final {
cc->Outputs()
.Tag("PRESENCE")
.AddPacket(MakePacket<bool>(!cc->Inputs().Tag("PACKET").IsEmpty())
.At(cc->InputTimestamp()));
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
};
REGISTER_CALCULATOR(PacketPresenceCalculator);
mediapipe/calculators/core/packet_resampler_calculator.cc
+17 -17
View File
@@ -47,7 +47,7 @@ TimestampDiff TimestampDiffFromSeconds(double seconds) {
}
} // namespace
::mediapipe::Status PacketResamplerCalculator::GetContract(
mediapipe::Status PacketResamplerCalculator::GetContract(
CalculatorContract* cc) {
const auto& resampler_options =
cc->Options<PacketResamplerCalculatorOptions>();
@@ -78,10 +78,10 @@ TimestampDiff TimestampDiffFromSeconds(double seconds) {
RET_CHECK(cc->InputSidePackets().HasTag("SEED"));
cc->InputSidePackets().Tag("SEED").Set<std::string>();
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status PacketResamplerCalculator::Open(CalculatorContext* cc) {
mediapipe::Status PacketResamplerCalculator::Open(CalculatorContext* cc) {
const auto resampler_options =
tool::RetrieveOptions(cc->Options<PacketResamplerCalculatorOptions>(),
cc->InputSidePackets(), "OPTIONS");
@@ -156,8 +156,8 @@ TimestampDiff TimestampDiffFromSeconds(double seconds) {
const auto& seed = cc->InputSidePackets().Tag("SEED").Get<std::string>();
random_ = CreateSecureRandom(seed);
if (random_ == nullptr) {
return ::mediapipe::Status(
::mediapipe::StatusCode::kInvalidArgument,
return mediapipe::Status(
mediapipe::StatusCode::kInvalidArgument,
"SecureRandom is not available. With \"jitter\" specified, "
"PacketResamplerCalculator processing cannot proceed.");
}
@@ -165,17 +165,17 @@ TimestampDiff TimestampDiffFromSeconds(double seconds) {
}
packet_reservoir_ =
std::make_unique<PacketReservoir>(packet_reservoir_random_.get());
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status PacketResamplerCalculator::Process(CalculatorContext* cc) {
mediapipe::Status PacketResamplerCalculator::Process(CalculatorContext* cc) {
if (cc->InputTimestamp() == Timestamp::PreStream() &&
cc->Inputs().UsesTags() && cc->Inputs().HasTag("VIDEO_HEADER") &&
!cc->Inputs().Tag("VIDEO_HEADER").IsEmpty()) {
video_header_ = cc->Inputs().Tag("VIDEO_HEADER").Get<VideoHeader>();
video_header_.frame_rate = frame_rate_;
if (cc->Inputs().Get(input_data_id_).IsEmpty()) {
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
}
if (jitter_ != 0.0 && random_ != nullptr) {
@@ -192,7 +192,7 @@ TimestampDiff TimestampDiffFromSeconds(double seconds) {
MP_RETURN_IF_ERROR(ProcessWithoutJitter(cc));
}
last_packet_ = cc->Inputs().Get(input_data_id_).Value();
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
void PacketResamplerCalculator::InitializeNextOutputTimestampWithJitter() {
@@ -229,7 +229,7 @@ void PacketResamplerCalculator::UpdateNextOutputTimestampWithJitter() {
((1.0 - jitter_) + 2.0 * jitter_ * random_->RandFloat());
}
::mediapipe::Status PacketResamplerCalculator::ProcessWithJitter(
mediapipe::Status PacketResamplerCalculator::ProcessWithJitter(
CalculatorContext* cc) {
RET_CHECK_GT(cc->InputTimestamp(), Timestamp::PreStream());
RET_CHECK_NE(jitter_, 0.0);
@@ -243,7 +243,7 @@ void PacketResamplerCalculator::UpdateNextOutputTimestampWithJitter() {
cc->Inputs().Get(input_data_id_).Value().At(next_output_timestamp_));
UpdateNextOutputTimestampWithJitter();
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
if (frame_time_usec_ <
@@ -267,10 +267,10 @@ void PacketResamplerCalculator::UpdateNextOutputTimestampWithJitter() {
.At(next_output_timestamp_));
UpdateNextOutputTimestampWithJitter();
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status PacketResamplerCalculator::ProcessWithoutJitter(
mediapipe::Status PacketResamplerCalculator::ProcessWithoutJitter(
CalculatorContext* cc) {
RET_CHECK_GT(cc->InputTimestamp(), Timestamp::PreStream());
RET_CHECK_EQ(jitter_, 0.0);
@@ -333,12 +333,12 @@ void PacketResamplerCalculator::UpdateNextOutputTimestampWithJitter() {
.Get(output_data_id_)
.SetNextTimestampBound(PeriodIndexToTimestamp(period_count_));
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status PacketResamplerCalculator::Close(CalculatorContext* cc) {
mediapipe::Status PacketResamplerCalculator::Close(CalculatorContext* cc) {
if (!cc->GraphStatus().ok()) {
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
// Emit the last packet received if we have at least one packet, but
// haven't sent anything for its period.
@@ -350,7 +350,7 @@ void PacketResamplerCalculator::UpdateNextOutputTimestampWithJitter() {
if (!packet_reservoir_->IsEmpty()) {
OutputWithinLimits(cc, packet_reservoir_->GetSample());
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
Timestamp PacketResamplerCalculator::PeriodIndexToTimestamp(int64 index) const {
mediapipe/calculators/core/packet_resampler_calculator.h
+6 -6
View File
@@ -99,11 +99,11 @@ class PacketReservoir {
// packet_downsampler_calculator.cc: skips packets regardless of timestamps.
class PacketResamplerCalculator : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc);
static mediapipe::Status GetContract(CalculatorContract* cc);
::mediapipe::Status Open(CalculatorContext* cc) override;
::mediapipe::Status Close(CalculatorContext* cc) override;
::mediapipe::Status Process(CalculatorContext* cc) override;
mediapipe::Status Open(CalculatorContext* cc) override;
mediapipe::Status Close(CalculatorContext* cc) override;
mediapipe::Status Process(CalculatorContext* cc) override;
private:
// Calculates the first sampled timestamp that incorporates a jittering
@@ -113,10 +113,10 @@ class PacketResamplerCalculator : public CalculatorBase {
void UpdateNextOutputTimestampWithJitter();
// Logic for Process() when jitter_ != 0.0.
::mediapipe::Status ProcessWithJitter(CalculatorContext* cc);
mediapipe::Status ProcessWithJitter(CalculatorContext* cc);
// Logic for Process() when jitter_ == 0.0.
::mediapipe::Status ProcessWithoutJitter(CalculatorContext* cc);
mediapipe::Status ProcessWithoutJitter(CalculatorContext* cc);
// Given the current count of periods that have passed, this returns
// the next valid timestamp of the middle point of the next period:
mediapipe/calculators/core/packet_thinner_calculator.cc
+17 -17
View File
@@ -90,7 +90,7 @@ class PacketThinnerCalculator : public CalculatorBase {
PacketThinnerCalculator() {}
~PacketThinnerCalculator() override {}
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
static mediapipe::Status GetContract(CalculatorContract* cc) {
if (cc->InputSidePackets().HasTag(kOptionsTag)) {
cc->InputSidePackets().Tag(kOptionsTag).Set<CalculatorOptions>();
}
@@ -99,21 +99,21 @@ class PacketThinnerCalculator : public CalculatorBase {
if (cc->InputSidePackets().HasTag(kPeriodTag)) {
cc->InputSidePackets().Tag(kPeriodTag).Set<int64>();
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status Open(CalculatorContext* cc) override;
::mediapipe::Status Close(CalculatorContext* cc) override;
::mediapipe::Status Process(CalculatorContext* cc) override {
mediapipe::Status Open(CalculatorContext* cc) override;
mediapipe::Status Close(CalculatorContext* cc) override;
mediapipe::Status Process(CalculatorContext* cc) override {
if (cc->InputTimestamp() < start_time_) {
return ::mediapipe::OkStatus(); // Drop packets before start_time_.
return mediapipe::OkStatus(); // Drop packets before start_time_.
} else if (cc->InputTimestamp() >= end_time_) {
if (!cc->Outputs().Index(0).IsClosed()) {
cc->Outputs()
.Index(0)
.Close(); // No more Packets will be output after end_time_.
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
} else {
return thinner_type_ == PacketThinnerCalculatorOptions::ASYNC
? AsyncThinnerProcess(cc)
@@ -123,8 +123,8 @@ class PacketThinnerCalculator : public CalculatorBase {
private:
// Implementation of ASYNC and SYNC versions of thinner algorithm.
::mediapipe::Status AsyncThinnerProcess(CalculatorContext* cc);
::mediapipe::Status SyncThinnerProcess(CalculatorContext* cc);
mediapipe::Status AsyncThinnerProcess(CalculatorContext* cc);
mediapipe::Status SyncThinnerProcess(CalculatorContext* cc);
// Cached option.
PacketThinnerCalculatorOptions::ThinnerType thinner_type_;
@@ -153,7 +153,7 @@ namespace {
TimestampDiff abs(TimestampDiff t) { return t < 0 ? -t : t; }
} // namespace
::mediapipe::Status PacketThinnerCalculator::Open(CalculatorContext* cc) {
mediapipe::Status PacketThinnerCalculator::Open(CalculatorContext* cc) {
PacketThinnerCalculatorOptions options = mediapipe::tool::RetrieveOptions(
cc->Options<PacketThinnerCalculatorOptions>(), cc->InputSidePackets(),
kOptionsTag);
@@ -224,10 +224,10 @@ TimestampDiff abs(TimestampDiff t) { return t < 0 ? -t : t; }
}
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status PacketThinnerCalculator::Close(CalculatorContext* cc) {
mediapipe::Status PacketThinnerCalculator::Close(CalculatorContext* cc) {
// Emit any saved packets before quitting.
if (!saved_packet_.IsEmpty()) {
// Only sync thinner should have saved packets.
@@ -239,10 +239,10 @@ TimestampDiff abs(TimestampDiff t) { return t < 0 ? -t : t; }
cc->Outputs().Index(0).AddPacket(saved_packet_);
}
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status PacketThinnerCalculator::AsyncThinnerProcess(
mediapipe::Status PacketThinnerCalculator::AsyncThinnerProcess(
CalculatorContext* cc) {
if (cc->InputTimestamp() >= next_valid_timestamp_) {
cc->Outputs().Index(0).AddPacket(
@@ -251,10 +251,10 @@ TimestampDiff abs(TimestampDiff t) { return t < 0 ? -t : t; }
// Guaranteed not to emit packets seen during refractory period.
cc->Outputs().Index(0).SetNextTimestampBound(next_valid_timestamp_);
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status PacketThinnerCalculator::SyncThinnerProcess(
mediapipe::Status PacketThinnerCalculator::SyncThinnerProcess(
CalculatorContext* cc) {
if (saved_packet_.IsEmpty()) {
// If no packet has been saved, store the current packet.
@@ -290,7 +290,7 @@ TimestampDiff abs(TimestampDiff t) { return t < 0 ? -t : t; }
saved_packet_ = cc->Inputs().Index(0).Value();
}
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
Timestamp PacketThinnerCalculator::NearestSyncTimestamp(Timestamp now) const {
mediapipe/calculators/core/pass_through_calculator.cc
+8 -8
View File
@@ -28,9 +28,9 @@ namespace mediapipe {
// ignored.
class PassThroughCalculator : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
static mediapipe::Status GetContract(CalculatorContract* cc) {
if (!cc->Inputs().TagMap()->SameAs(*cc->Outputs().TagMap())) {
return ::mediapipe::InvalidArgumentError(
return mediapipe::InvalidArgumentError(
"Input and output streams to PassThroughCalculator must use "
"matching tags and indexes.");
}
@@ -46,7 +46,7 @@ class PassThroughCalculator : public CalculatorBase {
if (cc->OutputSidePackets().NumEntries() != 0) {
if (!cc->InputSidePackets().TagMap()->SameAs(
*cc->OutputSidePackets().TagMap())) {
return ::mediapipe::InvalidArgumentError(
return mediapipe::InvalidArgumentError(
"Input and output side packets to PassThroughCalculator must use "
"matching tags and indexes.");
}
@@ -56,10 +56,10 @@ class PassThroughCalculator : public CalculatorBase {
&cc->InputSidePackets().Get(id));
}
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status Open(CalculatorContext* cc) final {
mediapipe::Status Open(CalculatorContext* cc) final {
for (CollectionItemId id = cc->Inputs().BeginId();
id < cc->Inputs().EndId(); ++id) {
if (!cc->Inputs().Get(id).Header().IsEmpty()) {
@@ -73,10 +73,10 @@ class PassThroughCalculator : public CalculatorBase {
}
}
cc->SetOffset(TimestampDiff(0));
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status Process(CalculatorContext* cc) final {
mediapipe::Status Process(CalculatorContext* cc) final {
cc->GetCounter("PassThrough")->Increment();
if (cc->Inputs().NumEntries() == 0) {
return tool::StatusStop();
@@ -90,7 +90,7 @@ class PassThroughCalculator : public CalculatorBase {
cc->Outputs().Get(id).AddPacket(cc->Inputs().Get(id).Value());
}
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
};
REGISTER_CALCULATOR(PassThroughCalculator);
mediapipe/calculators/core/previous_loopback_calculator.cc
+6 -6
View File
@@ -53,7 +53,7 @@ namespace mediapipe {
// }
class PreviousLoopbackCalculator : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
static mediapipe::Status GetContract(CalculatorContract* cc) {
cc->Inputs().Get("MAIN", 0).SetAny();
cc->Inputs().Get("LOOP", 0).SetAny();
cc->Outputs().Get("PREV_LOOP", 0).SetSameAs(&(cc->Inputs().Get("LOOP", 0)));
@@ -63,20 +63,20 @@ class PreviousLoopbackCalculator : public CalculatorBase {
// Process() function is invoked in response to MAIN/LOOP stream timestamp
// bound updates.
cc->SetProcessTimestampBounds(true);
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status Open(CalculatorContext* cc) final {
mediapipe::Status Open(CalculatorContext* cc) final {
main_id_ = cc->Inputs().GetId("MAIN", 0);
loop_id_ = cc->Inputs().GetId("LOOP", 0);
prev_loop_id_ = cc->Outputs().GetId("PREV_LOOP", 0);
cc->Outputs()
.Get(prev_loop_id_)
.SetHeader(cc->Inputs().Get(loop_id_).Header());
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status Process(CalculatorContext* cc) final {
mediapipe::Status Process(CalculatorContext* cc) final {
// Non-empty packets and empty packets indicating timestamp bound updates
// are guaranteed to have timestamps greater than timestamps of previous
// packets within the same stream. Calculator tracks and operates on such
@@ -139,7 +139,7 @@ class PreviousLoopbackCalculator : public CalculatorBase {
}
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
private:
mediapipe/calculators/core/previous_loopback_calculator_test.cc
+12 -12
View File
@@ -136,27 +136,27 @@ TEST(PreviousLoopbackCalculator, CorrectTimestamps) {
// A Calculator that outputs a summary packet in CalculatorBase::Close().
class PacketOnCloseCalculator : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
static mediapipe::Status GetContract(CalculatorContract* cc) {
cc->Inputs().Index(0).Set<int>();
cc->Outputs().Index(0).Set<int>();
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status Open(CalculatorContext* cc) final {
mediapipe::Status Open(CalculatorContext* cc) final {
cc->SetOffset(TimestampDiff(0));
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status Process(CalculatorContext* cc) final {
mediapipe::Status Process(CalculatorContext* cc) final {
sum_ += cc->Inputs().Index(0).Value().Get<int>();
cc->Outputs().Index(0).AddPacket(cc->Inputs().Index(0).Value());
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status Close(CalculatorContext* cc) final {
mediapipe::Status Close(CalculatorContext* cc) final {
cc->Outputs().Index(0).AddPacket(
MakePacket<int>(sum_).At(Timestamp::Max()));
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
private:
@@ -700,19 +700,19 @@ TEST_F(PreviousLoopbackCalculatorProcessingTimestampsTest,
// Similar to GateCalculator, but it doesn't propagate timestamp bound updates.
class DroppingGateCalculator : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
static mediapipe::Status GetContract(CalculatorContract* cc) {
cc->Inputs().Index(0).SetAny();
cc->Inputs().Tag("DISALLOW").Set<bool>();
cc->Outputs().Index(0).SetSameAs(&cc->Inputs().Index(0));
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status Process(CalculatorContext* cc) final {
mediapipe::Status Process(CalculatorContext* cc) final {
if (!cc->Inputs().Index(0).IsEmpty() &&
!cc->Inputs().Tag("DISALLOW").Get<bool>()) {
cc->Outputs().Index(0).AddPacket(cc->Inputs().Index(0).Value());
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
};
REGISTER_CALCULATOR(DroppingGateCalculator);
mediapipe/calculators/core/quantize_float_vector_calculator.cc
+8 -8
View File
@@ -43,32 +43,32 @@ namespace mediapipe {
class QuantizeFloatVectorCalculator : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
static mediapipe::Status GetContract(CalculatorContract* cc) {
cc->Inputs().Tag("FLOAT_VECTOR").Set<std::vector<float>>();
cc->Outputs().Tag("ENCODED").Set<std::string>();
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status Open(CalculatorContext* cc) final {
mediapipe::Status Open(CalculatorContext* cc) final {
const auto options =
cc->Options<::mediapipe::QuantizeFloatVectorCalculatorOptions>();
if (!options.has_max_quantized_value() ||
!options.has_min_quantized_value()) {
return ::mediapipe::InvalidArgumentError(
return mediapipe::InvalidArgumentError(
"Both max_quantized_value and min_quantized_value must be provided "
"in QuantizeFloatVectorCalculatorOptions.");
}
max_quantized_value_ = options.max_quantized_value();
min_quantized_value_ = options.min_quantized_value();
if (max_quantized_value_ < min_quantized_value_ + FLT_EPSILON) {
return ::mediapipe::InvalidArgumentError(
return mediapipe::InvalidArgumentError(
"max_quantized_value must be greater than min_quantized_value.");
}
range_ = max_quantized_value_ - min_quantized_value_;
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status Process(CalculatorContext* cc) final {
mediapipe::Status Process(CalculatorContext* cc) final {
const std::vector<float>& float_vector =
cc->Inputs().Tag("FLOAT_VECTOR").Value().Get<std::vector<float>>();
int feature_size = float_vector.size();
@@ -88,7 +88,7 @@ class QuantizeFloatVectorCalculator : public CalculatorBase {
}
cc->Outputs().Tag("ENCODED").AddPacket(
MakePacket<std::string>(encoded_features).At(cc->InputTimestamp()));
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
private:
mediapipe/calculators/core/real_time_flow_limiter_calculator.cc
+199
View File
@@ -0,0 +1,199 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <algorithm>
#include <utility>
#include <vector>
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/util/header_util.h"
namespace mediapipe {
// RealTimeFlowLimiterCalculator is used to limit the number of pipelined
// processing operations in a section of the graph.
//
// Typical topology:
//
// in ->-[FLC]-[foo]-...-[bar]-+->- out
// ^_____________________|
// FINISHED
//
// By connecting the output of the graph section to this calculator's FINISHED
// input with a backwards edge, this allows FLC to keep track of how many
// timestamps are currently being processed.
//
// The limit defaults to 1, and can be overridden with the MAX_IN_FLIGHT side
// packet.
//
// As long as the number of timestamps being processed ("in flight") is below
// the limit, FLC allows input to pass through. When the limit is reached,
// FLC starts dropping input packets, keeping only the most recent. When the
// processing count decreases again, as signaled by the receipt of a packet on
// FINISHED, FLC allows packets to flow again, releasing the most recently
// queued packet, if any.
//
// If there are multiple input streams, packet dropping is synchronized.
//
// IMPORTANT: for each timestamp where FLC forwards a packet (or a set of
// packets, if using multiple data streams), a packet must eventually arrive on
// the FINISHED stream. Dropping packets in the section between FLC and
// FINISHED will make the in-flight count incorrect.
//
// TODO: Remove this comment when graph-level ISH has been removed.
// NOTE: this calculator should always use the ImmediateInputStreamHandler and
// uses it by default. However, if the graph specifies a graph-level
// InputStreamHandler, to override that setting, the InputStreamHandler must
// be explicitly specified as shown below.
//
// Example config:
// node {
// calculator: "RealTimeFlowLimiterCalculator"
// input_stream: "raw_frames"
// input_stream: "FINISHED:finished"
// input_stream_info: {
// tag_index: 'FINISHED'
// back_edge: true
// }
// input_stream_handler {
// input_stream_handler: 'ImmediateInputStreamHandler'
// }
// output_stream: "gated_frames"
// }
class RealTimeFlowLimiterCalculator : public CalculatorBase {
public:
static mediapipe::Status GetContract(CalculatorContract* cc) {
int num_data_streams = cc->Inputs().NumEntries("");
RET_CHECK_GE(num_data_streams, 1);
RET_CHECK_EQ(cc->Outputs().NumEntries(""), num_data_streams)
<< "Output streams must correspond input streams except for the "
"finish indicator input stream.";
for (int i = 0; i < num_data_streams; ++i) {
cc->Inputs().Get("", i).SetAny();
cc->Outputs().Get("", i).SetSameAs(&(cc->Inputs().Get("", i)));
}
cc->Inputs().Get("FINISHED", 0).SetAny();
if (cc->InputSidePackets().HasTag("MAX_IN_FLIGHT")) {
cc->InputSidePackets().Tag("MAX_IN_FLIGHT").Set<int>();
}
if (cc->Outputs().HasTag("ALLOW")) {
cc->Outputs().Tag("ALLOW").Set<bool>();
}
cc->SetInputStreamHandler("ImmediateInputStreamHandler");
return mediapipe::OkStatus();
}
mediapipe::Status Open(CalculatorContext* cc) final {
finished_id_ = cc->Inputs().GetId("FINISHED", 0);
max_in_flight_ = 1;
if (cc->InputSidePackets().HasTag("MAX_IN_FLIGHT")) {
max_in_flight_ = cc->InputSidePackets().Tag("MAX_IN_FLIGHT").Get<int>();
}
RET_CHECK_GE(max_in_flight_, 1);
num_in_flight_ = 0;
allowed_id_ = cc->Outputs().GetId("ALLOW", 0);
allow_ctr_ts_ = Timestamp(0);
num_data_streams_ = cc->Inputs().NumEntries("");
data_stream_bound_ts_.resize(num_data_streams_);
RET_CHECK_OK(CopyInputHeadersToOutputs(cc->Inputs(), &(cc->Outputs())));
return mediapipe::OkStatus();
}
bool Allow() { return num_in_flight_ < max_in_flight_; }
mediapipe::Status Process(CalculatorContext* cc) final {
bool old_allow = Allow();
Timestamp lowest_incomplete_ts = Timestamp::Done();
// Process FINISHED stream.
if (!cc->Inputs().Get(finished_id_).Value().IsEmpty()) {
RET_CHECK_GT(num_in_flight_, 0)
<< "Received a FINISHED packet, but we had none in flight.";
--num_in_flight_;
}
// Process data streams.
for (int i = 0; i < num_data_streams_; ++i) {
auto& stream = cc->Inputs().Get("", i);
auto& out = cc->Outputs().Get("", i);
Packet& packet = stream.Value();
auto ts = packet.Timestamp();
if (ts.IsRangeValue() && data_stream_bound_ts_[i] <= ts) {
data_stream_bound_ts_[i] = ts + 1;
// Note: it's ok to update the output bound here, before sending the
// packet, because updates are batched during the Process function.
out.SetNextTimestampBound(data_stream_bound_ts_[i]);
}
lowest_incomplete_ts =
std::min(lowest_incomplete_ts, data_stream_bound_ts_[i]);
if (packet.IsEmpty()) {
// If the input stream is closed, close the corresponding output.
if (stream.IsDone() && !out.IsClosed()) {
out.Close();
}
// TODO: if the packet is empty, the ts is unset, and we
// cannot read the timestamp bound, even though we'd like to propagate
// it.
} else if (mediapipe::ContainsKey(pending_ts_, ts)) {
// If we have already sent this timestamp (on another stream), send it
// on this stream too.
out.AddPacket(std::move(packet));
} else if (Allow() && (ts > last_dropped_ts_)) {
// If the in-flight is under the limit, and if we have not already
// dropped this or a later timestamp on another stream, then send
// the packet and add an in-flight timestamp.
out.AddPacket(std::move(packet));
pending_ts_.insert(ts);
++num_in_flight_;
} else {
// Otherwise, we'll drop the packet.
last_dropped_ts_ = std::max(last_dropped_ts_, ts);
}
}
// Remove old pending_ts_ entries.
auto it = std::lower_bound(pending_ts_.begin(), pending_ts_.end(),
lowest_incomplete_ts);
pending_ts_.erase(pending_ts_.begin(), it);
// Update ALLOW signal.
if ((old_allow != Allow()) && allowed_id_.IsValid()) {
cc->Outputs()
.Get(allowed_id_)
.AddPacket(MakePacket<bool>(Allow()).At(++allow_ctr_ts_));
}
return mediapipe::OkStatus();
}
private:
std::set<Timestamp> pending_ts_;
Timestamp last_dropped_ts_;
int num_data_streams_;
int num_in_flight_;
int max_in_flight_;
CollectionItemId finished_id_;
CollectionItemId allowed_id_;
Timestamp allow_ctr_ts_;
std::vector<Timestamp> data_stream_bound_ts_;
};
REGISTER_CALCULATOR(RealTimeFlowLimiterCalculator);
} // namespace mediapipe
mediapipe/calculators/core/real_time_flow_limiter_calculator_test.cc
+496
View File
@@ -0,0 +1,496 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <algorithm>
#include <memory>
#include <string>
#include <vector>
#include "absl/time/clock.h"
#include "absl/time/time.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_runner.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/port/gmock.h"
#include "mediapipe/framework/port/gtest.h"
#include "mediapipe/framework/port/integral_types.h"
#include "mediapipe/framework/port/parse_text_proto.h"
#include "mediapipe/framework/port/status_matchers.h"
#include "mediapipe/framework/timestamp.h"
#include "mediapipe/framework/tool/sink.h"
namespace mediapipe {
namespace {
// A simple Semaphore for synchronizing test threads.
class AtomicSemaphore {
public:
AtomicSemaphore(int64_t supply) : supply_(supply) {}
void Acquire(int64_t amount) {
while (supply_.fetch_sub(amount) - amount < 0) {
Release(amount);
}
}
void Release(int64_t amount) { supply_.fetch_add(amount); }
private:
std::atomic<int64_t> supply_;
};
// Returns the timestamp values for a vector of Packets.
std::vector<int64> TimestampValues(const std::vector<Packet>& packets) {
std::vector<int64> result;
for (const Packet& packet : packets) {
result.push_back(packet.Timestamp().Value());
}
return result;
}
// Returns the packet values for a vector of Packets.
template <typename T>
std::vector<T> PacketValues(const std::vector<Packet>& packets) {
std::vector<T> result;
for (const Packet& packet : packets) {
result.push_back(packet.Get<T>());
}
return result;
}
constexpr int kNumImageFrames = 5;
constexpr int kNumFinished = 3;
CalculatorGraphConfig::Node GetDefaultNode() {
return ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
calculator: "RealTimeFlowLimiterCalculator"
input_stream: "raw_frames"
input_stream: "FINISHED:finished"
input_stream_info: { tag_index: "FINISHED" back_edge: true }
output_stream: "gated_frames"
)");
}
// Simple test to make sure that the RealTimeFlowLimiterCalculator outputs just
// one packet when MAX_IN_FLIGHT is 1.
TEST(RealTimeFlowLimiterCalculator, OneOutputTest) {
// Setup the calculator runner and add only ImageFrame packets.
CalculatorRunner runner(GetDefaultNode());
for (int i = 0; i < kNumImageFrames; ++i) {
Timestamp timestamp = Timestamp(i * Timestamp::kTimestampUnitsPerSecond);
runner.MutableInputs()->Index(0).packets.push_back(
MakePacket<ImageFrame>().At(timestamp));
}
// Run the calculator.
MP_ASSERT_OK(runner.Run()) << "Calculator execution failed.";
const std::vector<Packet>& frame_output_packets =
runner.Outputs().Index(0).packets;
EXPECT_EQ(frame_output_packets.size(), 1);
}
// Simple test to make sure that the RealTimeFlowLimiterCalculator waits for all
// input streams to have at least one packet available before publishing.
TEST(RealTimeFlowLimiterCalculator, BasicTest) {
// Setup the calculator runner and add both ImageFrame and finish packets.
CalculatorRunner runner(GetDefaultNode());
for (int i = 0; i < kNumImageFrames; ++i) {
Timestamp timestamp = Timestamp(i * Timestamp::kTimestampUnitsPerSecond);
runner.MutableInputs()->Index(0).packets.push_back(
MakePacket<ImageFrame>().At(timestamp));
}
for (int i = 0; i < kNumFinished; ++i) {
Timestamp timestamp =
Timestamp((i + 1) * Timestamp::kTimestampUnitsPerSecond);
runner.MutableInputs()
->Tag("FINISHED")
.packets.push_back(MakePacket<bool>(true).At(timestamp));
}
// Run the calculator.
MP_ASSERT_OK(runner.Run()) << "Calculator execution failed.";
const std::vector<Packet>& frame_output_packets =
runner.Outputs().Index(0).packets;
// Only outputs packets if both input streams are available.
int expected_num_packets = std::min(kNumImageFrames, kNumFinished + 1);
EXPECT_EQ(frame_output_packets.size(), expected_num_packets);
}
// A Calculator::Process callback function.
typedef std::function<mediapipe::Status(const InputStreamShardSet&,
OutputStreamShardSet*)>
ProcessFunction;
// A testing callback function that passes through all packets.
mediapipe::Status PassthroughFunction(const InputStreamShardSet& inputs,
OutputStreamShardSet* outputs) {
for (int i = 0; i < inputs.NumEntries(); ++i) {
if (!inputs.Index(i).Value().IsEmpty()) {
outputs->Index(i).AddPacket(inputs.Index(i).Value());
}
}
return mediapipe::OkStatus();
}
// A Calculator that runs a testing callback function in Close.
class CloseCallbackCalculator : public CalculatorBase {
public:
static mediapipe::Status GetContract(CalculatorContract* cc) {
for (CollectionItemId id = cc->Inputs().BeginId();
id < cc->Inputs().EndId(); ++id) {
cc->Inputs().Get(id).SetAny();
}
for (CollectionItemId id = cc->Outputs().BeginId();
id < cc->Outputs().EndId(); ++id) {
cc->Outputs().Get(id).SetAny();
}
cc->InputSidePackets().Index(0).Set<std::function<mediapipe::Status()>>();
return mediapipe::OkStatus();
}
mediapipe::Status Process(CalculatorContext* cc) override {
return PassthroughFunction(cc->Inputs(), &(cc->Outputs()));
}
mediapipe::Status Close(CalculatorContext* cc) override {
const auto& callback = cc->InputSidePackets()
.Index(0)
.Get<std::function<mediapipe::Status()>>();
return callback();
}
};
REGISTER_CALCULATOR(CloseCallbackCalculator);
// Tests demostrating an RealTimeFlowLimiterCalculator operating in a cyclic
// graph.
// TODO: clean up these tests.
class RealTimeFlowLimiterCalculatorTest : public testing::Test {
public:
RealTimeFlowLimiterCalculatorTest()
: enter_semaphore_(0), exit_semaphore_(0) {}
void SetUp() override {
graph_config_ = InflightGraphConfig();
tool::AddVectorSink("out_1", &graph_config_, &out_1_packets_);
tool::AddVectorSink("out_2", &graph_config_, &out_2_packets_);
}
void InitializeGraph(int max_in_flight) {
ProcessFunction semaphore_0_func = [&](const InputStreamShardSet& inputs,
OutputStreamShardSet* outputs) {
enter_semaphore_.Release(1);
return PassthroughFunction(inputs, outputs);
};
ProcessFunction semaphore_1_func = [&](const InputStreamShardSet& inputs,
OutputStreamShardSet* outputs) {
exit_semaphore_.Acquire(1);
return PassthroughFunction(inputs, outputs);
};
std::function<mediapipe::Status()> close_func = [this]() {
close_count_++;
return mediapipe::OkStatus();
};
MP_ASSERT_OK(graph_.Initialize(
graph_config_, {
{"max_in_flight", MakePacket<int>(max_in_flight)},
{"callback_0", Adopt(new auto(semaphore_0_func))},
{"callback_1", Adopt(new auto(semaphore_1_func))},
{"callback_2", Adopt(new auto(close_func))},
}));
}
// Adds a packet to a graph input stream.
void AddPacket(const std::string& input_name, int value) {
MP_EXPECT_OK(graph_.AddPacketToInputStream(
input_name, MakePacket<int>(value).At(Timestamp(value))));
}
// A calculator graph starting with an RealTimeFlowLimiterCalculator and
// ending with a InFlightFinishCalculator.
// Back-edge "finished" limits processing to one frame in-flight.
// The two LambdaCalculators are used to keep certain packet sets in flight.
CalculatorGraphConfig InflightGraphConfig() {
return ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
input_stream: 'in_1'
input_stream: 'in_2'
node {
calculator: 'RealTimeFlowLimiterCalculator'
input_side_packet: 'MAX_IN_FLIGHT:max_in_flight'
input_stream: 'in_1'
input_stream: 'in_2'
input_stream: 'FINISHED:out_1'
input_stream_info: { tag_index: 'FINISHED' back_edge: true }
output_stream: 'in_1_sampled'
output_stream: 'in_2_sampled'
}
node {
calculator: 'LambdaCalculator'
input_side_packet: 'callback_0'
input_stream: 'in_1_sampled'
input_stream: 'in_2_sampled'
output_stream: 'queue_1'
output_stream: 'queue_2'
}
node {
calculator: 'LambdaCalculator'
input_side_packet: 'callback_1'
input_stream: 'queue_1'
input_stream: 'queue_2'
output_stream: 'close_1'
output_stream: 'close_2'
}
node {
calculator: 'CloseCallbackCalculator'
input_side_packet: 'callback_2'
input_stream: 'close_1'
input_stream: 'close_2'
output_stream: 'out_1'
output_stream: 'out_2'
}
)");
}
protected:
CalculatorGraphConfig graph_config_;
CalculatorGraph graph_;
AtomicSemaphore enter_semaphore_;
AtomicSemaphore exit_semaphore_;
std::vector<Packet> out_1_packets_;
std::vector<Packet> out_2_packets_;
int close_count_ = 0;
};
// A test demonstrating an RealTimeFlowLimiterCalculator operating in a cyclic
// graph. This test shows that:
//
// (1) Timestamps are passed through unaltered.
// (2) All output streams including the back_edge stream are closed when
// the first input stream is closed.
//
TEST_F(RealTimeFlowLimiterCalculatorTest, BackEdgeCloses) {
InitializeGraph(1);
MP_ASSERT_OK(graph_.StartRun({}));
auto send_packet = [this](const std::string& input_name, int64 n) {
MP_EXPECT_OK(graph_.AddPacketToInputStream(
input_name, MakePacket<int64>(n).At(Timestamp(n))));
};
for (int i = 0; i < 10; i++) {
send_packet("in_1", i * 10);
// This next input should be dropped.
send_packet("in_1", i * 10 + 5);
MP_EXPECT_OK(graph_.WaitUntilIdle());
send_packet("in_2", i * 10);
exit_semaphore_.Release(1);
MP_EXPECT_OK(graph_.WaitUntilIdle());
}
MP_EXPECT_OK(graph_.CloseInputStream("in_1"));
MP_EXPECT_OK(graph_.CloseInputStream("in_2"));
MP_EXPECT_OK(graph_.WaitUntilIdle());
// All output streams are closed and all output packets are delivered,
// with stream "in_1" and stream "in_2" closed.
EXPECT_EQ(10, out_1_packets_.size());
EXPECT_EQ(10, out_2_packets_.size());
// Timestamps have not been messed with.
EXPECT_EQ(PacketValues<int64>(out_1_packets_),
TimestampValues(out_1_packets_));
EXPECT_EQ(PacketValues<int64>(out_2_packets_),
TimestampValues(out_2_packets_));
// Extra inputs on in_1 have been dropped
EXPECT_EQ(TimestampValues(out_1_packets_),
(std::vector<int64>{0, 10, 20, 30, 40, 50, 60, 70, 80, 90}));
EXPECT_EQ(TimestampValues(out_1_packets_), TimestampValues(out_2_packets_));
// The closing of the stream has been propagated.
EXPECT_EQ(1, close_count_);
}
// A test demonstrating that all output streams are closed when all
// input streams are closed after the last input packet has been processed.
TEST_F(RealTimeFlowLimiterCalculatorTest, AllStreamsClose) {
InitializeGraph(1);
MP_ASSERT_OK(graph_.StartRun({}));
exit_semaphore_.Release(10);
for (int i = 0; i < 10; i++) {
AddPacket("in_1", i);
MP_EXPECT_OK(graph_.WaitUntilIdle());
AddPacket("in_2", i);
MP_EXPECT_OK(graph_.WaitUntilIdle());
}
MP_EXPECT_OK(graph_.CloseAllInputStreams());
MP_EXPECT_OK(graph_.WaitUntilIdle());
EXPECT_EQ(TimestampValues(out_1_packets_), TimestampValues(out_2_packets_));
EXPECT_EQ(TimestampValues(out_1_packets_),
(std::vector<int64>{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}));
EXPECT_EQ(1, close_count_);
}
TEST(RealTimeFlowLimiterCalculator, TwoStreams) {
std::vector<Packet> a_passed;
std::vector<Packet> b_passed;
CalculatorGraphConfig graph_config_ =
ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
input_stream: 'in_a'
input_stream: 'in_b'
input_stream: 'finished'
node {
name: 'input_dropper'
calculator: 'RealTimeFlowLimiterCalculator'
input_side_packet: 'MAX_IN_FLIGHT:max_in_flight'
input_stream: 'in_a'
input_stream: 'in_b'
input_stream: 'FINISHED:finished'
input_stream_info: { tag_index: 'FINISHED' back_edge: true }
output_stream: 'in_a_sampled'
output_stream: 'in_b_sampled'
output_stream: 'ALLOW:allow'
}
)");
std::string allow_cb_name;
tool::AddVectorSink("in_a_sampled", &graph_config_, &a_passed);
tool::AddVectorSink("in_b_sampled", &graph_config_, &b_passed);
tool::AddCallbackCalculator("allow", &graph_config_, &allow_cb_name, true);
bool allow = true;
auto allow_cb = [&allow](const Packet& packet) {
allow = packet.Get<bool>();
};
CalculatorGraph graph_;
MP_EXPECT_OK(graph_.Initialize(
graph_config_,
{
{"max_in_flight", MakePacket<int>(1)},
{allow_cb_name,
MakePacket<std::function<void(const Packet&)>>(allow_cb)},
}));
MP_EXPECT_OK(graph_.StartRun({}));
auto send_packet = [&graph_](const std::string& input_name, int n) {
MP_EXPECT_OK(graph_.AddPacketToInputStream(
input_name, MakePacket<int>(n).At(Timestamp(n))));
};
send_packet("in_a", 1);
MP_EXPECT_OK(graph_.WaitUntilIdle());
EXPECT_EQ(allow, false);
EXPECT_EQ(TimestampValues(a_passed), (std::vector<int64>{1}));
EXPECT_EQ(TimestampValues(b_passed), (std::vector<int64>{}));
send_packet("in_a", 2);
send_packet("in_b", 1);
MP_EXPECT_OK(graph_.WaitUntilIdle());
EXPECT_EQ(TimestampValues(a_passed), (std::vector<int64>{1}));
EXPECT_EQ(TimestampValues(b_passed), (std::vector<int64>{1}));
EXPECT_EQ(allow, false);
send_packet("finished", 1);
MP_EXPECT_OK(graph_.WaitUntilIdle());
EXPECT_EQ(TimestampValues(a_passed), (std::vector<int64>{1}));
EXPECT_EQ(TimestampValues(b_passed), (std::vector<int64>{1}));
EXPECT_EQ(allow, true);
send_packet("in_b", 2);
MP_EXPECT_OK(graph_.WaitUntilIdle());
EXPECT_EQ(TimestampValues(a_passed), (std::vector<int64>{1}));
EXPECT_EQ(TimestampValues(b_passed), (std::vector<int64>{1}));
EXPECT_EQ(allow, true);
send_packet("in_b", 3);
MP_EXPECT_OK(graph_.WaitUntilIdle());
EXPECT_EQ(TimestampValues(a_passed), (std::vector<int64>{1}));
EXPECT_EQ(TimestampValues(b_passed), (std::vector<int64>{1, 3}));
EXPECT_EQ(allow, false);
send_packet("in_b", 4);
MP_EXPECT_OK(graph_.WaitUntilIdle());
EXPECT_EQ(TimestampValues(a_passed), (std::vector<int64>{1}));
EXPECT_EQ(TimestampValues(b_passed), (std::vector<int64>{1, 3}));
EXPECT_EQ(allow, false);
send_packet("in_a", 3);
MP_EXPECT_OK(graph_.WaitUntilIdle());
EXPECT_EQ(TimestampValues(a_passed), (std::vector<int64>{1, 3}));
EXPECT_EQ(TimestampValues(b_passed), (std::vector<int64>{1, 3}));
EXPECT_EQ(allow, false);
send_packet("finished", 3);
MP_EXPECT_OK(graph_.WaitUntilIdle());
EXPECT_EQ(TimestampValues(a_passed), (std::vector<int64>{1, 3}));
EXPECT_EQ(TimestampValues(b_passed), (std::vector<int64>{1, 3}));
EXPECT_EQ(allow, true);
MP_EXPECT_OK(graph_.CloseAllInputStreams());
MP_EXPECT_OK(graph_.WaitUntilDone());
}
TEST(RealTimeFlowLimiterCalculator, CanConsume) {
std::vector<Packet> in_sampled_packets_;
CalculatorGraphConfig graph_config_ =
ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
input_stream: 'in'
input_stream: 'finished'
node {
name: 'input_dropper'
calculator: 'RealTimeFlowLimiterCalculator'
input_side_packet: 'MAX_IN_FLIGHT:max_in_flight'
input_stream: 'in'
input_stream: 'FINISHED:finished'
input_stream_info: { tag_index: 'FINISHED' back_edge: true }
output_stream: 'in_sampled'
output_stream: 'ALLOW:allow'
}
)");
std::string allow_cb_name;
tool::AddVectorSink("in_sampled", &graph_config_, &in_sampled_packets_);
tool::AddCallbackCalculator("allow", &graph_config_, &allow_cb_name, true);
bool allow = true;
auto allow_cb = [&allow](const Packet& packet) {
allow = packet.Get<bool>();
};
CalculatorGraph graph_;
MP_EXPECT_OK(graph_.Initialize(
graph_config_,
{
{"max_in_flight", MakePacket<int>(1)},
{allow_cb_name,
MakePacket<std::function<void(const Packet&)>>(allow_cb)},
}));
MP_EXPECT_OK(graph_.StartRun({}));
auto send_packet = [&graph_](const std::string& input_name, int n) {
MP_EXPECT_OK(graph_.AddPacketToInputStream(
input_name, MakePacket<int>(n).At(Timestamp(n))));
};
send_packet("in", 1);
MP_EXPECT_OK(graph_.WaitUntilIdle());
EXPECT_EQ(allow, false);
EXPECT_EQ(TimestampValues(in_sampled_packets_), (std::vector<int64>{1}));
MP_EXPECT_OK(in_sampled_packets_[0].Consume<int>());
MP_EXPECT_OK(graph_.CloseAllInputStreams());
MP_EXPECT_OK(graph_.WaitUntilDone());
}
} // anonymous namespace
} // namespace mediapipe
mediapipe/calculators/core/round_robin_demux_calculator.cc
+6 -6
View File
@@ -73,7 +73,7 @@ namespace mediapipe {
// MuxCalculator/MuxInputStreamHandler.
class RoundRobinDemuxCalculator : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
static mediapipe::Status GetContract(CalculatorContract* cc) {
RET_CHECK_EQ(cc->Inputs().NumEntries(), 1);
cc->Inputs().Index(0).SetAny();
if (cc->Outputs().HasTag("SELECT")) {
@@ -83,18 +83,18 @@ class RoundRobinDemuxCalculator : public CalculatorBase {
id < cc->Outputs().EndId("OUTPUT"); ++id) {
cc->Outputs().Get(id).SetSameAs(&cc->Inputs().Index(0));
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status Open(CalculatorContext* cc) override {
mediapipe::Status Open(CalculatorContext* cc) override {
select_output_ = cc->Outputs().GetId("SELECT", 0);
output_data_stream_index_ = 0;
output_data_stream_base_ = cc->Outputs().GetId("OUTPUT", 0);
num_output_data_streams_ = cc->Outputs().NumEntries("OUTPUT");
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status Process(CalculatorContext* cc) override {
mediapipe::Status Process(CalculatorContext* cc) override {
cc->Outputs()
.Get(output_data_stream_base_ + output_data_stream_index_)
.AddPacket(cc->Inputs().Index(0).Value());
@@ -105,7 +105,7 @@ class RoundRobinDemuxCalculator : public CalculatorBase {
}
output_data_stream_index_ =
(output_data_stream_index_ + 1) % num_output_data_streams_;
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
private:
mediapipe/calculators/core/sequence_shift_calculator.cc
+8 -8
View File
@@ -30,18 +30,18 @@ namespace mediapipe {
// second, and so on.
class SequenceShiftCalculator : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
static mediapipe::Status GetContract(CalculatorContract* cc) {
cc->Inputs().Index(0).SetAny();
if (cc->InputSidePackets().HasTag(kPacketOffsetTag)) {
cc->InputSidePackets().Tag(kPacketOffsetTag).Set<int>();
}
cc->Outputs().Index(0).SetSameAs(&cc->Inputs().Index(0));
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
// Reads from options to set cache_size_ and packet_offset_.
::mediapipe::Status Open(CalculatorContext* cc) override;
::mediapipe::Status Process(CalculatorContext* cc) override;
mediapipe::Status Open(CalculatorContext* cc) override;
mediapipe::Status Process(CalculatorContext* cc) override;
private:
static constexpr const char* kPacketOffsetTag = "PACKET_OFFSET";
@@ -72,7 +72,7 @@ class SequenceShiftCalculator : public CalculatorBase {
};
REGISTER_CALCULATOR(SequenceShiftCalculator);
::mediapipe::Status SequenceShiftCalculator::Open(CalculatorContext* cc) {
mediapipe::Status SequenceShiftCalculator::Open(CalculatorContext* cc) {
packet_offset_ =
cc->Options<mediapipe::SequenceShiftCalculatorOptions>().packet_offset();
if (cc->InputSidePackets().HasTag(kPacketOffsetTag)) {
@@ -83,10 +83,10 @@ REGISTER_CALCULATOR(SequenceShiftCalculator);
if (packet_offset_ == 0) {
cc->Outputs().Index(0).SetOffset(0);
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status SequenceShiftCalculator::Process(CalculatorContext* cc) {
mediapipe::Status SequenceShiftCalculator::Process(CalculatorContext* cc) {
if (packet_offset_ > 0) {
ProcessPositiveOffset(cc);
} else if (packet_offset_ < 0) {
@@ -94,7 +94,7 @@ REGISTER_CALCULATOR(SequenceShiftCalculator);
} else {
cc->Outputs().Index(0).AddPacket(cc->Inputs().Index(0).Value());
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
void SequenceShiftCalculator::ProcessPositiveOffset(CalculatorContext* cc) {
mediapipe/calculators/core/side_packet_to_stream_calculator.cc
+13 -14
View File
@@ -89,10 +89,10 @@ class SidePacketToStreamCalculator : public CalculatorBase {
SidePacketToStreamCalculator() = default;
~SidePacketToStreamCalculator() override = default;
static ::mediapipe::Status GetContract(CalculatorContract* cc);
::mediapipe::Status Open(CalculatorContext* cc) override;
::mediapipe::Status Process(CalculatorContext* cc) override;
::mediapipe::Status Close(CalculatorContext* cc) override;
static mediapipe::Status GetContract(CalculatorContract* cc);
mediapipe::Status Open(CalculatorContext* cc) override;
mediapipe::Status Process(CalculatorContext* cc) override;
mediapipe::Status Close(CalculatorContext* cc) override;
private:
bool is_tick_processing_ = false;
@@ -100,7 +100,7 @@ class SidePacketToStreamCalculator : public CalculatorBase {
};
REGISTER_CALCULATOR(SidePacketToStreamCalculator);
::mediapipe::Status SidePacketToStreamCalculator::GetContract(
mediapipe::Status SidePacketToStreamCalculator::GetContract(
CalculatorContract* cc) {
const auto& tags = cc->Outputs().GetTags();
RET_CHECK(tags.size() == 1 && kTimestampMap->count(*tags.begin()) == 1)
@@ -138,10 +138,10 @@ REGISTER_CALCULATOR(SidePacketToStreamCalculator);
cc->Inputs().Tag(kTagTick).SetAny();
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status SidePacketToStreamCalculator::Open(CalculatorContext* cc) {
mediapipe::Status SidePacketToStreamCalculator::Open(CalculatorContext* cc) {
output_tag_ = GetOutputTag(*cc);
if (cc->Inputs().HasTag(kTagTick)) {
is_tick_processing_ = true;
@@ -149,11 +149,10 @@ REGISTER_CALCULATOR(SidePacketToStreamCalculator);
// timestamp bound update.
cc->SetOffset(TimestampDiff(0));
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status SidePacketToStreamCalculator::Process(
CalculatorContext* cc) {
mediapipe::Status SidePacketToStreamCalculator::Process(CalculatorContext* cc) {
if (is_tick_processing_) {
// TICK input is guaranteed to be non-empty, as it's the only input stream
// for this calculator.
@@ -164,13 +163,13 @@ REGISTER_CALCULATOR(SidePacketToStreamCalculator);
.AddPacket(cc->InputSidePackets().Index(i).At(timestamp));
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
return ::mediapipe::tool::StatusStop();
return mediapipe::tool::StatusStop();
}
::mediapipe::Status SidePacketToStreamCalculator::Close(CalculatorContext* cc) {
mediapipe::Status SidePacketToStreamCalculator::Close(CalculatorContext* cc) {
if (!cc->Outputs().HasTag(kTagAtTick) &&
!cc->Outputs().HasTag(kTagAtTimestamp)) {
const auto& timestamp = kTimestampMap->at(output_tag_);
@@ -188,7 +187,7 @@ REGISTER_CALCULATOR(SidePacketToStreamCalculator);
.AddPacket(cc->InputSidePackets().Index(i).At(Timestamp(timestamp)));
}
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
} // namespace mediapipe
mediapipe/calculators/core/side_packet_to_stream_calculator_test.cc
+29 -21
View File
@@ -20,6 +20,7 @@
#include "absl/strings/str_replace.h"
#include "absl/strings/string_view.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/port/gmock.h"
#include "mediapipe/framework/port/gtest.h"
#include "mediapipe/framework/port/integral_types.h"
#include "mediapipe/framework/port/parse_text_proto.h"
@@ -30,6 +31,8 @@
namespace mediapipe {
namespace {
using testing::HasSubstr;
TEST(SidePacketToStreamCalculator, WrongConfig_MissingTick) {
CalculatorGraphConfig graph_config =
ParseTextProtoOrDie<CalculatorGraphConfig>(
@@ -46,9 +49,10 @@ TEST(SidePacketToStreamCalculator, WrongConfig_MissingTick) {
CalculatorGraph graph;
auto status = graph.Initialize(graph_config);
EXPECT_FALSE(status.ok());
EXPECT_PRED2(
absl::StrContains, status.message(),
"Either both of TICK and AT_TICK should be used or none of them.");
EXPECT_THAT(
status.message(),
HasSubstr(
"Either both of TICK and AT_TICK should be used or none of them."));
}
TEST(SidePacketToStreamCalculator, WrongConfig_MissingTimestampSideInput) {
@@ -67,9 +71,9 @@ TEST(SidePacketToStreamCalculator, WrongConfig_MissingTimestampSideInput) {
CalculatorGraph graph;
auto status = graph.Initialize(graph_config);
EXPECT_FALSE(status.ok());
EXPECT_PRED2(
absl::StrContains, status.message(),
"Either both TIMESTAMP and AT_TIMESTAMP should be used or none of them.");
EXPECT_THAT(status.message(),
HasSubstr("Either both TIMESTAMP and AT_TIMESTAMP should be used "
"or none of them."));
}
TEST(SidePacketToStreamCalculator, WrongConfig_NonExistentTag) {
@@ -88,10 +92,11 @@ TEST(SidePacketToStreamCalculator, WrongConfig_NonExistentTag) {
CalculatorGraph graph;
auto status = graph.Initialize(graph_config);
EXPECT_FALSE(status.ok());
EXPECT_PRED2(absl::StrContains, status.message(),
"Only one of AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO, AT_TICK and "
"AT_TIMESTAMP tags is allowed and required to specify output "
"stream(s).");
EXPECT_THAT(
status.message(),
HasSubstr("Only one of AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO, AT_TICK and "
"AT_TIMESTAMP tags is allowed and required to specify output "
"stream(s)."));
}
TEST(SidePacketToStreamCalculator, WrongConfig_MixedTags) {
@@ -112,10 +117,11 @@ TEST(SidePacketToStreamCalculator, WrongConfig_MixedTags) {
CalculatorGraph graph;
auto status = graph.Initialize(graph_config);
EXPECT_FALSE(status.ok());
EXPECT_PRED2(absl::StrContains, status.message(),
"Only one of AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO, AT_TICK and "
"AT_TIMESTAMP tags is allowed and required to specify output "
"stream(s).");
EXPECT_THAT(
status.message(),
HasSubstr("Only one of AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO, AT_TICK and "
"AT_TIMESTAMP tags is allowed and required to specify output "
"stream(s)."));
}
TEST(SidePacketToStreamCalculator, WrongConfig_NotEnoughSidePackets) {
@@ -134,9 +140,10 @@ TEST(SidePacketToStreamCalculator, WrongConfig_NotEnoughSidePackets) {
CalculatorGraph graph;
auto status = graph.Initialize(graph_config);
EXPECT_FALSE(status.ok());
EXPECT_PRED2(
absl::StrContains, status.message(),
"Same number of input side packets and output streams is required.");
EXPECT_THAT(
status.message(),
HasSubstr(
"Same number of input side packets and output streams is required."));
}
TEST(SidePacketToStreamCalculator, WrongConfig_NotEnoughOutputStreams) {
@@ -155,9 +162,10 @@ TEST(SidePacketToStreamCalculator, WrongConfig_NotEnoughOutputStreams) {
CalculatorGraph graph;
auto status = graph.Initialize(graph_config);
EXPECT_FALSE(status.ok());
EXPECT_PRED2(
absl::StrContains, status.message(),
"Same number of input side packets and output streams is required.");
EXPECT_THAT(
status.message(),
HasSubstr(
"Same number of input side packets and output streams is required."));
}
void DoTestNonAtTickOutputTag(absl::string_view tag,
@@ -181,7 +189,7 @@ void DoTestNonAtTickOutputTag(absl::string_view tag,
MP_ASSERT_OK(graph.ObserveOutputStream(
"packet", [&output_packets](const Packet& packet) {
output_packets.push_back(packet);
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}));
MP_ASSERT_OK(
graph.StartRun({{"side_packet", MakePacket<int>(expected_value)}}));
mediapipe/calculators/core/split_normalized_landmark_list_calculator.cc
+10 -10
View File
@@ -35,7 +35,7 @@ namespace mediapipe {
// NormalizedLandmarkList.
class SplitNormalizedLandmarkListCalculator : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
static mediapipe::Status GetContract(CalculatorContract* cc) {
RET_CHECK(cc->Inputs().NumEntries() == 1);
RET_CHECK(cc->Outputs().NumEntries() != 0);
@@ -55,7 +55,7 @@ class SplitNormalizedLandmarkListCalculator : public CalculatorBase {
range_0.begin() < range_1.end()) ||
(range_1.begin() >= range_0.begin() &&
range_1.begin() < range_0.end())) {
return ::mediapipe::InvalidArgumentError(
return mediapipe::InvalidArgumentError(
"Ranges must be non-overlapping when using combine_outputs "
"option.");
}
@@ -63,7 +63,7 @@ class SplitNormalizedLandmarkListCalculator : public CalculatorBase {
}
} else {
if (cc->Outputs().NumEntries() != options.ranges_size()) {
return ::mediapipe::InvalidArgumentError(
return mediapipe::InvalidArgumentError(
"The number of output streams should match the number of ranges "
"specified in the CalculatorOptions.");
}
@@ -72,13 +72,13 @@ class SplitNormalizedLandmarkListCalculator : public CalculatorBase {
for (int i = 0; i < cc->Outputs().NumEntries(); ++i) {
if (options.ranges(i).begin() < 0 || options.ranges(i).end() < 0 ||
options.ranges(i).begin() >= options.ranges(i).end()) {
return ::mediapipe::InvalidArgumentError(
return mediapipe::InvalidArgumentError(
"Indices should be non-negative and begin index should be less "
"than the end index.");
}
if (options.element_only()) {
if (options.ranges(i).end() - options.ranges(i).begin() != 1) {
return ::mediapipe::InvalidArgumentError(
return mediapipe::InvalidArgumentError(
"Since element_only is true, all ranges should be of size 1.");
}
cc->Outputs().Index(i).Set<NormalizedLandmark>();
@@ -88,10 +88,10 @@ class SplitNormalizedLandmarkListCalculator : public CalculatorBase {
}
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status Open(CalculatorContext* cc) override {
mediapipe::Status Open(CalculatorContext* cc) override {
cc->SetOffset(TimestampDiff(0));
const auto& options =
@@ -106,10 +106,10 @@ class SplitNormalizedLandmarkListCalculator : public CalculatorBase {
total_elements_ += range.end() - range.begin();
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status Process(CalculatorContext* cc) override {
mediapipe::Status Process(CalculatorContext* cc) override {
const NormalizedLandmarkList& input =
cc->Inputs().Index(0).Get<NormalizedLandmarkList>();
RET_CHECK_GE(input.landmark_size(), max_range_end_)
@@ -148,7 +148,7 @@ class SplitNormalizedLandmarkListCalculator : public CalculatorBase {
}
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
private:
mediapipe/calculators/core/split_normalized_landmark_list_calculator_test.cc
+8 -8
View File
@@ -121,7 +121,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest, SmokeTest) {
// Prepare a graph to use the SplitNormalizedLandmarkListCalculator.
CalculatorGraphConfig graph_config =
::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
R"(
input_stream: "landmarks_in"
node {
@@ -170,7 +170,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest, SmokeTest) {
TEST_F(SplitNormalizedLandmarkListCalculatorTest, InvalidRangeTest) {
// Prepare a graph to use the SplitNormalizedLandmarkListCalculator.
CalculatorGraphConfig graph_config =
::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
R"(
input_stream: "landmarks_in"
node {
@@ -195,7 +195,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest,
InvalidOutputStreamCountTest) {
// Prepare a graph to use the SplitNormalizedLandmarkListCalculator.
CalculatorGraphConfig graph_config =
::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
R"(
input_stream: "landmarks_in"
node {
@@ -222,7 +222,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest,
InvalidCombineOutputsMultipleOutputsTest) {
// Prepare a graph to use the SplitNormalizedLandmarkListCalculator.
CalculatorGraphConfig graph_config =
::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
R"(
input_stream: "landmarks_in"
node {
@@ -251,7 +251,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest,
InvalidOverlappingRangesTest) {
// Prepare a graph to use the SplitNormalizedLandmarkListCalculator.
CalculatorGraphConfig graph_config =
::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
R"(
input_stream: "landmarks_in"
node {
@@ -280,7 +280,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest, SmokeTestElementOnly) {
// Prepare a graph to use the SplitNormalizedLandmarkListCalculator.
CalculatorGraphConfig graph_config =
::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
R"(
input_stream: "landmarks_in"
node {
@@ -333,7 +333,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest, SmokeTestCombiningOutputs) {
// Prepare a graph to use the SplitNormalizedLandmarkListCalculator.
CalculatorGraphConfig graph_config =
::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
R"(
input_stream: "landmarks_in"
node {
@@ -376,7 +376,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest,
ElementOnlyDisablesVectorOutputs) {
// Prepare a graph to use the SplitNormalizedLandmarkListCalculator.
CalculatorGraphConfig graph_config =
::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
R"(
input_stream: "landmarks_in"
node {
mediapipe/calculators/core/split_vector_calculator.h
+21 -21
View File
@@ -58,7 +58,7 @@ using IsNotMovable =
template <typename T, bool move_elements>
class SplitVectorCalculator : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
static mediapipe::Status GetContract(CalculatorContract* cc) {
RET_CHECK(cc->Inputs().NumEntries() == 1);
RET_CHECK(cc->Outputs().NumEntries() != 0);
@@ -79,7 +79,7 @@ class SplitVectorCalculator : public CalculatorBase {
RET_CHECK_OK(checkRangesDontOverlap(options));
} else {
if (cc->Outputs().NumEntries() != options.ranges_size()) {
return ::mediapipe::InvalidArgumentError(
return mediapipe::InvalidArgumentError(
"The number of output streams should match the number of ranges "
"specified in the CalculatorOptions.");
}
@@ -88,13 +88,13 @@ class SplitVectorCalculator : public CalculatorBase {
for (int i = 0; i < cc->Outputs().NumEntries(); ++i) {
if (options.ranges(i).begin() < 0 || options.ranges(i).end() < 0 ||
options.ranges(i).begin() >= options.ranges(i).end()) {
return ::mediapipe::InvalidArgumentError(
return mediapipe::InvalidArgumentError(
"Indices should be non-negative and begin index should be less "
"than the end index.");
}
if (options.element_only()) {
if (options.ranges(i).end() - options.ranges(i).begin() != 1) {
return ::mediapipe::InvalidArgumentError(
return mediapipe::InvalidArgumentError(
"Since element_only is true, all ranges should be of size 1.");
}
cc->Outputs().Index(i).Set<T>();
@@ -104,10 +104,10 @@ class SplitVectorCalculator : public CalculatorBase {
}
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status Open(CalculatorContext* cc) override {
mediapipe::Status Open(CalculatorContext* cc) override {
cc->SetOffset(TimestampDiff(0));
const auto& options =
@@ -122,11 +122,11 @@ class SplitVectorCalculator : public CalculatorBase {
total_elements_ += range.end() - range.begin();
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status Process(CalculatorContext* cc) override {
if (cc->Inputs().Index(0).IsEmpty()) return ::mediapipe::OkStatus();
mediapipe::Status Process(CalculatorContext* cc) override {
if (cc->Inputs().Index(0).IsEmpty()) return mediapipe::OkStatus();
if (move_elements) {
return ProcessMovableElements<T>(cc);
@@ -136,7 +136,7 @@ class SplitVectorCalculator : public CalculatorBase {
}
template <typename U, IsCopyable<U> = true>
::mediapipe::Status ProcessCopyableElements(CalculatorContext* cc) {
mediapipe::Status ProcessCopyableElements(CalculatorContext* cc) {
// static_assert(std::is_copy_constructible<U>::value,
// "Cannot copy non-copyable elements");
const auto& input = cc->Inputs().Index(0).Get<std::vector<U>>();
@@ -167,17 +167,17 @@ class SplitVectorCalculator : public CalculatorBase {
}
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
template <typename U, IsNotCopyable<U> = true>
::mediapipe::Status ProcessCopyableElements(CalculatorContext* cc) {
return ::mediapipe::InternalError("Cannot copy non-copyable elements.");
mediapipe::Status ProcessCopyableElements(CalculatorContext* cc) {
return mediapipe::InternalError("Cannot copy non-copyable elements.");
}
template <typename U, IsMovable<U> = true>
::mediapipe::Status ProcessMovableElements(CalculatorContext* cc) {
::mediapipe::StatusOr<std::unique_ptr<std::vector<U>>> input_status =
mediapipe::Status ProcessMovableElements(CalculatorContext* cc) {
mediapipe::StatusOr<std::unique_ptr<std::vector<U>>> input_status =
cc->Inputs().Index(0).Value().Consume<std::vector<U>>();
if (!input_status.ok()) return input_status.status();
std::unique_ptr<std::vector<U>> input_vector =
@@ -214,16 +214,16 @@ class SplitVectorCalculator : public CalculatorBase {
}
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
template <typename U, IsNotMovable<U> = true>
::mediapipe::Status ProcessMovableElements(CalculatorContext* cc) {
return ::mediapipe::InternalError("Cannot move non-movable elements.");
mediapipe::Status ProcessMovableElements(CalculatorContext* cc) {
return mediapipe::InternalError("Cannot move non-movable elements.");
}
private:
static ::mediapipe::Status checkRangesDontOverlap(
static mediapipe::Status checkRangesDontOverlap(
const ::mediapipe::SplitVectorCalculatorOptions& options) {
for (int i = 0; i < options.ranges_size() - 1; ++i) {
for (int j = i + 1; j < options.ranges_size(); ++j) {
@@ -233,13 +233,13 @@ class SplitVectorCalculator : public CalculatorBase {
range_0.begin() < range_1.end()) ||
(range_1.begin() >= range_0.begin() &&
range_1.begin() < range_0.end())) {
return ::mediapipe::InvalidArgumentError(
return mediapipe::InvalidArgumentError(
"Ranges must be non-overlapping when using combine_outputs "
"option.");
}
}
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
std::vector<std::pair<int32, int32>> ranges_;
mediapipe/calculators/core/split_vector_calculator_test.cc
+12 -12
View File
@@ -162,7 +162,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest, SmokeTest) {
// Prepare a graph to use the SplitTfLiteTensorVectorCalculator.
CalculatorGraphConfig graph_config =
::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
R"(
input_stream: "tensor_in"
node {
@@ -213,7 +213,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest, InvalidRangeTest) {
// Prepare a graph to use the SplitTfLiteTensorVectorCalculator.
CalculatorGraphConfig graph_config =
::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
R"(
input_stream: "tensor_in"
node {
@@ -239,7 +239,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest, InvalidOutputStreamCountTest) {
// Prepare a graph to use the SplitTfLiteTensorVectorCalculator.
CalculatorGraphConfig graph_config =
::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
R"(
input_stream: "tensor_in"
node {
@@ -268,7 +268,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest,
// Prepare a graph to use the SplitTfLiteTensorVectorCalculator.
CalculatorGraphConfig graph_config =
::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
R"(
input_stream: "tensor_in"
node {
@@ -298,7 +298,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest, InvalidOverlappingRangesTest) {
// Prepare a graph to use the SplitTfLiteTensorVectorCalculator.
CalculatorGraphConfig graph_config =
::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
R"(
input_stream: "tensor_in"
node {
@@ -329,7 +329,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest, SmokeTestElementOnly) {
// Prepare a graph to use the SplitTfLiteTensorVectorCalculator.
CalculatorGraphConfig graph_config =
::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
R"(
input_stream: "tensor_in"
node {
@@ -384,7 +384,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest, SmokeTestCombiningOutputs) {
// Prepare a graph to use the SplitTfLiteTensorVectorCalculator.
CalculatorGraphConfig graph_config =
::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
R"(
input_stream: "tensor_in"
node {
@@ -427,7 +427,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest,
ElementOnlyDisablesVectorOutputs) {
// Prepare a graph to use the SplitTfLiteTensorVectorCalculator.
CalculatorGraphConfig graph_config =
::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
R"(
input_stream: "tensor_in"
node {
@@ -510,7 +510,7 @@ class MovableSplitUniqueIntPtrCalculatorTest : public ::testing::Test {
TEST_F(MovableSplitUniqueIntPtrCalculatorTest, InvalidOverlappingRangesTest) {
// Prepare a graph to use the TestMovableSplitUniqueIntPtrVectorCalculator.
CalculatorGraphConfig graph_config =
::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
R"(
input_stream: "input_vector"
node {
@@ -535,7 +535,7 @@ TEST_F(MovableSplitUniqueIntPtrCalculatorTest, InvalidOverlappingRangesTest) {
TEST_F(MovableSplitUniqueIntPtrCalculatorTest, SmokeTest) {
// Prepare a graph to use the TestMovableSplitUniqueIntPtrVectorCalculator.
CalculatorGraphConfig graph_config =
::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
R"(
input_stream: "input_vector"
node {
@@ -591,7 +591,7 @@ TEST_F(MovableSplitUniqueIntPtrCalculatorTest, SmokeTest) {
TEST_F(MovableSplitUniqueIntPtrCalculatorTest, SmokeTestElementOnly) {
// Prepare a graph to use the TestMovableSplitUniqueIntPtrVectorCalculator.
CalculatorGraphConfig graph_config =
::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
R"(
input_stream: "input_vector"
node {
@@ -645,7 +645,7 @@ TEST_F(MovableSplitUniqueIntPtrCalculatorTest, SmokeTestElementOnly) {
TEST_F(MovableSplitUniqueIntPtrCalculatorTest, SmokeTestCombiningOutputs) {
// Prepare a graph to use the TestMovableSplitUniqueIntPtrVectorCalculator.
CalculatorGraphConfig graph_config =
::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
R"(
input_stream: "input_vector"
node {
mediapipe/calculators/core/string_to_int_calculator.cc
+7 -7
View File
@@ -36,25 +36,25 @@ namespace mediapipe {
template <typename IntType>
class StringToIntCalculatorTemplate : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
static mediapipe::Status GetContract(CalculatorContract* cc) {
cc->InputSidePackets().Index(0).Set<std::string>();
cc->OutputSidePackets().Index(0).Set<IntType>();
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status Open(CalculatorContext* cc) override {
mediapipe::Status Open(CalculatorContext* cc) override {
IntType number;
if (!absl::SimpleAtoi(cc->InputSidePackets().Index(0).Get<std::string>(),
&number)) {
return ::mediapipe::InvalidArgumentError(
return mediapipe::InvalidArgumentError(
"The std::string could not be parsed as an integer.");
}
cc->OutputSidePackets().Index(0).Set(MakePacket<IntType>(number));
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status Process(CalculatorContext* cc) override {
return ::mediapipe::OkStatus();
mediapipe::Status Process(CalculatorContext* cc) override {
return mediapipe::OkStatus();
}
};
mediapipe/calculators/image/bilateral_filter_calculator.cc
+29 -31
View File
@@ -82,18 +82,18 @@ class BilateralFilterCalculator : public CalculatorBase {
BilateralFilterCalculator() = default;
~BilateralFilterCalculator() override = default;
static ::mediapipe::Status GetContract(CalculatorContract* cc);
static mediapipe::Status GetContract(CalculatorContract* cc);
// From Calculator.
::mediapipe::Status Open(CalculatorContext* cc) override;
::mediapipe::Status Process(CalculatorContext* cc) override;
::mediapipe::Status Close(CalculatorContext* cc) override;
mediapipe::Status Open(CalculatorContext* cc) override;
mediapipe::Status Process(CalculatorContext* cc) override;
mediapipe::Status Close(CalculatorContext* cc) override;
private:
::mediapipe::Status RenderGpu(CalculatorContext* cc);
::mediapipe::Status RenderCpu(CalculatorContext* cc);
mediapipe::Status RenderGpu(CalculatorContext* cc);
mediapipe::Status RenderCpu(CalculatorContext* cc);
::mediapipe::Status GlSetup(CalculatorContext* cc);
mediapipe::Status GlSetup(CalculatorContext* cc);
void GlRender(CalculatorContext* cc);
mediapipe::BilateralFilterCalculatorOptions options_;
@@ -111,17 +111,17 @@ class BilateralFilterCalculator : public CalculatorBase {
};
REGISTER_CALCULATOR(BilateralFilterCalculator);
::mediapipe::Status BilateralFilterCalculator::GetContract(
mediapipe::Status BilateralFilterCalculator::GetContract(
CalculatorContract* cc) {
CHECK_GE(cc->Inputs().NumEntries(), 1);
if (cc->Inputs().HasTag(kInputFrameTag) &&
cc->Inputs().HasTag(kInputFrameTagGpu)) {
return ::mediapipe::InternalError("Cannot have multiple input images.");
return mediapipe::InternalError("Cannot have multiple input images.");
}
if (cc->Inputs().HasTag(kInputFrameTagGpu) !=
cc->Outputs().HasTag(kOutputFrameTagGpu)) {
return ::mediapipe::InternalError("GPU output must have GPU input.");
return mediapipe::InternalError("GPU output must have GPU input.");
}
bool use_gpu = false;
@@ -165,10 +165,10 @@ REGISTER_CALCULATOR(BilateralFilterCalculator);
#endif // !MEDIAPIPE_DISABLE_GPU
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status BilateralFilterCalculator::Open(CalculatorContext* cc) {
mediapipe::Status BilateralFilterCalculator::Open(CalculatorContext* cc) {
cc->SetOffset(TimestampDiff(0));
options_ = cc->Options<mediapipe::BilateralFilterCalculatorOptions>();
@@ -194,30 +194,30 @@ REGISTER_CALCULATOR(BilateralFilterCalculator);
#endif // !MEDIAPIPE_DISABLE_GPU
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status BilateralFilterCalculator::Process(CalculatorContext* cc) {
mediapipe::Status BilateralFilterCalculator::Process(CalculatorContext* cc) {
if (use_gpu_) {
#if !defined(MEDIAPIPE_DISABLE_GPU)
MP_RETURN_IF_ERROR(
gpu_helper_.RunInGlContext([this, cc]() -> ::mediapipe::Status {
gpu_helper_.RunInGlContext([this, cc]() -> mediapipe::Status {
if (!gpu_initialized_) {
MP_RETURN_IF_ERROR(GlSetup(cc));
gpu_initialized_ = true;
}
MP_RETURN_IF_ERROR(RenderGpu(cc));
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}));
#endif // !MEDIAPIPE_DISABLE_GPU
} else {
MP_RETURN_IF_ERROR(RenderCpu(cc));
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status BilateralFilterCalculator::Close(CalculatorContext* cc) {
mediapipe::Status BilateralFilterCalculator::Close(CalculatorContext* cc) {
#if !defined(MEDIAPIPE_DISABLE_GPU)
gpu_helper_.RunInGlContext([this] {
if (program_) glDeleteProgram(program_);
@@ -230,13 +230,12 @@ REGISTER_CALCULATOR(BilateralFilterCalculator);
});
#endif // !MEDIAPIPE_DISABLE_GPU
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status BilateralFilterCalculator::RenderCpu(
CalculatorContext* cc) {
mediapipe::Status BilateralFilterCalculator::RenderCpu(CalculatorContext* cc) {
if (cc->Inputs().Tag(kInputFrameTag).IsEmpty()) {
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
const auto& input_frame = cc->Inputs().Tag(kInputFrameTag).Get<ImageFrame>();
@@ -244,7 +243,7 @@ REGISTER_CALCULATOR(BilateralFilterCalculator);
// Only 1 or 3 channel images supported by OpenCV.
if ((input_mat.channels() == 1 || input_mat.channels() == 3)) {
return ::mediapipe::InternalError(
return mediapipe::InternalError(
"CPU filtering supports only 1 or 3 channel input images.");
}
@@ -255,7 +254,7 @@ REGISTER_CALCULATOR(BilateralFilterCalculator);
if (has_guide_image) {
// cv::jointBilateralFilter() is in contrib module 'ximgproc'.
return ::mediapipe::UnimplementedError(
return mediapipe::UnimplementedError(
"CPU joint filtering support is not implemented yet.");
} else {
auto output_mat = mediapipe::formats::MatView(output_frame.get());
@@ -267,13 +266,12 @@ REGISTER_CALCULATOR(BilateralFilterCalculator);
cc->Outputs()
.Tag(kOutputFrameTag)
.Add(output_frame.release(), cc->InputTimestamp());
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status BilateralFilterCalculator::RenderGpu(
CalculatorContext* cc) {
mediapipe::Status BilateralFilterCalculator::RenderGpu(CalculatorContext* cc) {
if (cc->Inputs().Tag(kInputFrameTagGpu).IsEmpty()) {
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
#if !defined(MEDIAPIPE_DISABLE_GPU)
const auto& input_frame =
@@ -334,7 +332,7 @@ REGISTER_CALCULATOR(BilateralFilterCalculator);
output_texture.Release();
#endif // !MEDIAPIPE_DISABLE_GPU
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
void BilateralFilterCalculator::GlRender(CalculatorContext* cc) {
@@ -350,7 +348,7 @@ void BilateralFilterCalculator::GlRender(CalculatorContext* cc) {
#endif // !MEDIAPIPE_DISABLE_GPU
}
::mediapipe::Status BilateralFilterCalculator::GlSetup(CalculatorContext* cc) {
mediapipe::Status BilateralFilterCalculator::GlSetup(CalculatorContext* cc) {
#if !defined(MEDIAPIPE_DISABLE_GPU)
const GLint attr_location[NUM_ATTRIBUTES] = {
ATTRIB_VERTEX,
@@ -517,7 +515,7 @@ void BilateralFilterCalculator::GlRender(CalculatorContext* cc) {
#endif // !MEDIAPIPE_DISABLE_GPU
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
} // namespace mediapipe
mediapipe/calculators/image/color_convert_calculator.cc
+15 -16
View File
@@ -78,12 +78,12 @@ constexpr char kGrayOutTag[] = "GRAY_OUT";
class ColorConvertCalculator : public CalculatorBase {
public:
~ColorConvertCalculator() override = default;
static ::mediapipe::Status GetContract(CalculatorContract* cc);
::mediapipe::Status Process(CalculatorContext* cc) override;
static mediapipe::Status GetContract(CalculatorContract* cc);
mediapipe::Status Process(CalculatorContext* cc) override;
::mediapipe::Status Open(CalculatorContext* cc) override {
mediapipe::Status Open(CalculatorContext* cc) override {
cc->SetOffset(TimestampDiff(0));
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
private:
@@ -91,17 +91,16 @@ class ColorConvertCalculator : public CalculatorBase {
// conversion. The ImageFrame on input_tag is converted using the
// open_cv_convert_code provided and then output on the output_tag stream.
// Note that the output_format must match the destination conversion code.
::mediapipe::Status ConvertAndOutput(const std::string& input_tag,
const std::string& output_tag,
ImageFormat::Format output_format,
int open_cv_convert_code,
CalculatorContext* cc);
mediapipe::Status ConvertAndOutput(const std::string& input_tag,
const std::string& output_tag,
ImageFormat::Format output_format,
int open_cv_convert_code,
CalculatorContext* cc);
};
REGISTER_CALCULATOR(ColorConvertCalculator);
::mediapipe::Status ColorConvertCalculator::GetContract(
CalculatorContract* cc) {
mediapipe::Status ColorConvertCalculator::GetContract(CalculatorContract* cc) {
RET_CHECK_EQ(cc->Inputs().NumEntries(), 1)
<< "Only one input stream is allowed.";
RET_CHECK_EQ(cc->Outputs().NumEntries(), 1)
@@ -139,10 +138,10 @@ REGISTER_CALCULATOR(ColorConvertCalculator);
cc->Outputs().Tag(kBgraOutTag).Set<ImageFrame>();
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status ColorConvertCalculator::ConvertAndOutput(
mediapipe::Status ColorConvertCalculator::ConvertAndOutput(
const std::string& input_tag, const std::string& output_tag,
ImageFormat::Format output_format, int open_cv_convert_code,
CalculatorContext* cc) {
@@ -161,10 +160,10 @@ REGISTER_CALCULATOR(ColorConvertCalculator);
cc->Outputs()
.Tag(output_tag)
.Add(output_frame.release(), cc->InputTimestamp());
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status ColorConvertCalculator::Process(CalculatorContext* cc) {
mediapipe::Status ColorConvertCalculator::Process(CalculatorContext* cc) {
// RGBA -> RGB
if (cc->Inputs().HasTag(kRgbaInTag) && cc->Outputs().HasTag(kRgbOutTag)) {
return ConvertAndOutput(kRgbaInTag, kRgbOutTag, ImageFormat::SRGB,
@@ -196,7 +195,7 @@ REGISTER_CALCULATOR(ColorConvertCalculator);
cv::COLOR_RGBA2BGRA, cc);
}
return ::mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
return mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
<< "Unsupported image format conversion.";
}
mediapipe/calculators/image/feature_detector_calculator.cc
+12 -12
View File
@@ -50,15 +50,15 @@ class FeatureDetectorCalculator : public CalculatorBase {
public:
~FeatureDetectorCalculator() override = default;
static ::mediapipe::Status GetContract(CalculatorContract* cc);
static mediapipe::Status GetContract(CalculatorContract* cc);
::mediapipe::Status Open(CalculatorContext* cc) override;
::mediapipe::Status Process(CalculatorContext* cc) override;
mediapipe::Status Open(CalculatorContext* cc) override;
mediapipe::Status Process(CalculatorContext* cc) override;
private:
FeatureDetectorCalculatorOptions options_;
cv::Ptr<cv::Feature2D> feature_detector_;
std::unique_ptr<::mediapipe::ThreadPool> pool_;
std::unique_ptr<mediapipe::ThreadPool> pool_;
// Create image pyramid based on input image.
void ComputeImagePyramid(const cv::Mat& input_image,
@@ -71,7 +71,7 @@ class FeatureDetectorCalculator : public CalculatorBase {
REGISTER_CALCULATOR(FeatureDetectorCalculator);
::mediapipe::Status FeatureDetectorCalculator::GetContract(
mediapipe::Status FeatureDetectorCalculator::GetContract(
CalculatorContract* cc) {
if (cc->Inputs().HasTag("IMAGE")) {
cc->Inputs().Tag("IMAGE").Set<ImageFrame>();
@@ -85,26 +85,26 @@ REGISTER_CALCULATOR(FeatureDetectorCalculator);
if (cc->Outputs().HasTag("PATCHES")) {
cc->Outputs().Tag("PATCHES").Set<std::vector<TfLiteTensor>>();
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status FeatureDetectorCalculator::Open(CalculatorContext* cc) {
mediapipe::Status FeatureDetectorCalculator::Open(CalculatorContext* cc) {
options_ =
tool::RetrieveOptions(cc->Options(), cc->InputSidePackets(), kOptionsTag)
.GetExtension(FeatureDetectorCalculatorOptions::ext);
feature_detector_ = cv::ORB::create(
options_.max_features(), options_.scale_factor(),
options_.pyramid_level(), kPatchSize - 1, 0, 2, cv::ORB::FAST_SCORE);
pool_ = absl::make_unique<::mediapipe::ThreadPool>("ThreadPool", kNumThreads);
pool_ = absl::make_unique<mediapipe::ThreadPool>("ThreadPool", kNumThreads);
pool_->StartWorkers();
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status FeatureDetectorCalculator::Process(CalculatorContext* cc) {
mediapipe::Status FeatureDetectorCalculator::Process(CalculatorContext* cc) {
const Timestamp& timestamp = cc->InputTimestamp();
if (timestamp == Timestamp::PreStream()) {
// Indicator packet.
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
InputStream* input_frame = &(cc->Inputs().Tag("IMAGE"));
cv::Mat input_view = formats::MatView(&input_frame->Get<ImageFrame>());
@@ -176,7 +176,7 @@ REGISTER_CALCULATOR(FeatureDetectorCalculator);
cc->Outputs().Tag("PATCHES").Add(patches.release(), timestamp);
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
void FeatureDetectorCalculator::ComputeImagePyramid(
mediapipe/calculators/image/image_cropping_calculator.cc
+25 -26
View File
@@ -53,8 +53,7 @@ constexpr char kWidthTag[] = "WIDTH";
REGISTER_CALCULATOR(ImageCroppingCalculator);
::mediapipe::Status ImageCroppingCalculator::GetContract(
CalculatorContract* cc) {
mediapipe::Status ImageCroppingCalculator::GetContract(CalculatorContract* cc) {
RET_CHECK(cc->Inputs().HasTag(kImageTag) ^ cc->Inputs().HasTag(kImageGpuTag));
RET_CHECK(cc->Outputs().HasTag(kImageTag) ^
cc->Outputs().HasTag(kImageGpuTag));
@@ -116,10 +115,10 @@ REGISTER_CALCULATOR(ImageCroppingCalculator);
#endif // !MEDIAPIPE_DISABLE_GPU
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status ImageCroppingCalculator::Open(CalculatorContext* cc) {
mediapipe::Status ImageCroppingCalculator::Open(CalculatorContext* cc) {
cc->SetOffset(TimestampDiff(0));
if (cc->Inputs().HasTag(kImageGpuTag)) {
@@ -147,38 +146,38 @@ REGISTER_CALCULATOR(ImageCroppingCalculator);
MP_RETURN_IF_ERROR(ValidateBorderModeForCPU(cc));
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status ImageCroppingCalculator::Process(CalculatorContext* cc) {
mediapipe::Status ImageCroppingCalculator::Process(CalculatorContext* cc) {
if (cc->Inputs().HasTag(kRectTag) && cc->Inputs().Tag(kRectTag).IsEmpty()) {
VLOG(1) << "RECT is empty for timestamp: " << cc->InputTimestamp();
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
if (cc->Inputs().HasTag(kNormRectTag) &&
cc->Inputs().Tag(kNormRectTag).IsEmpty()) {
VLOG(1) << "NORM_RECT is empty for timestamp: " << cc->InputTimestamp();
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
if (use_gpu_) {
#if !defined(MEDIAPIPE_DISABLE_GPU)
MP_RETURN_IF_ERROR(
gpu_helper_.RunInGlContext([this, cc]() -> ::mediapipe::Status {
gpu_helper_.RunInGlContext([this, cc]() -> mediapipe::Status {
if (!gpu_initialized_) {
MP_RETURN_IF_ERROR(InitGpu(cc));
gpu_initialized_ = true;
}
MP_RETURN_IF_ERROR(RenderGpu(cc));
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}));
#endif // !MEDIAPIPE_DISABLE_GPU
} else {
MP_RETURN_IF_ERROR(RenderCpu(cc));
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status ImageCroppingCalculator::Close(CalculatorContext* cc) {
mediapipe::Status ImageCroppingCalculator::Close(CalculatorContext* cc) {
#if !defined(MEDIAPIPE_DISABLE_GPU)
gpu_helper_.RunInGlContext([this] {
if (program_) glDeleteProgram(program_);
@@ -187,16 +186,16 @@ REGISTER_CALCULATOR(ImageCroppingCalculator);
gpu_initialized_ = false;
#endif // !MEDIAPIPE_DISABLE_GPU
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status ImageCroppingCalculator::ValidateBorderModeForCPU(
mediapipe::Status ImageCroppingCalculator::ValidateBorderModeForCPU(
CalculatorContext* cc) {
int border_mode;
return GetBorderModeForOpenCV(cc, &border_mode);
}
::mediapipe::Status ImageCroppingCalculator::ValidateBorderModeForGPU(
mediapipe::Status ImageCroppingCalculator::ValidateBorderModeForGPU(
CalculatorContext* cc) {
mediapipe::ImageCroppingCalculatorOptions options =
cc->Options<mediapipe::ImageCroppingCalculatorOptions>();
@@ -213,12 +212,12 @@ REGISTER_CALCULATOR(ImageCroppingCalculator);
<< options.border_mode();
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status ImageCroppingCalculator::RenderCpu(CalculatorContext* cc) {
mediapipe::Status ImageCroppingCalculator::RenderCpu(CalculatorContext* cc) {
if (cc->Inputs().Tag(kImageTag).IsEmpty()) {
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
const auto& input_img = cc->Inputs().Tag(kImageTag).Get<ImageFrame>();
cv::Mat input_mat = formats::MatView(&input_img);
@@ -268,12 +267,12 @@ REGISTER_CALCULATOR(ImageCroppingCalculator);
cropped_image.copyTo(output_mat);
cc->Outputs().Tag(kImageTag).Add(output_frame.release(),
cc->InputTimestamp());
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status ImageCroppingCalculator::RenderGpu(CalculatorContext* cc) {
mediapipe::Status ImageCroppingCalculator::RenderGpu(CalculatorContext* cc) {
if (cc->Inputs().Tag(kImageGpuTag).IsEmpty()) {
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
#if !defined(MEDIAPIPE_DISABLE_GPU)
const Packet& input_packet = cc->Inputs().Tag(kImageGpuTag).Value();
@@ -308,7 +307,7 @@ REGISTER_CALCULATOR(ImageCroppingCalculator);
dst_tex.Release();
#endif // !MEDIAPIPE_DISABLE_GPU
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
void ImageCroppingCalculator::GlRender() {
@@ -359,7 +358,7 @@ void ImageCroppingCalculator::GlRender() {
#endif // !MEDIAPIPE_DISABLE_GPU
}
::mediapipe::Status ImageCroppingCalculator::InitGpu(CalculatorContext* cc) {
mediapipe::Status ImageCroppingCalculator::InitGpu(CalculatorContext* cc) {
#if !defined(MEDIAPIPE_DISABLE_GPU)
const GLint attr_location[NUM_ATTRIBUTES] = {
ATTRIB_VERTEX,
@@ -408,7 +407,7 @@ void ImageCroppingCalculator::GlRender() {
glUniform1i(glGetUniformLocation(program_, "input_frame"), 1);
#endif // !MEDIAPIPE_DISABLE_GPU
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
// For GPU only.
@@ -534,7 +533,7 @@ RectSpec ImageCroppingCalculator::GetCropSpecs(const CalculatorContext* cc,
return {crop_width, crop_height, x_center, y_center, rotation};
}
::mediapipe::Status ImageCroppingCalculator::GetBorderModeForOpenCV(
mediapipe::Status ImageCroppingCalculator::GetBorderModeForOpenCV(
CalculatorContext* cc, int* border_mode) {
mediapipe::ImageCroppingCalculatorOptions options =
cc->Options<mediapipe::ImageCroppingCalculatorOptions>();
@@ -551,7 +550,7 @@ RectSpec ImageCroppingCalculator::GetCropSpecs(const CalculatorContext* cc,
<< options.border_mode();
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
} // namespace mediapipe
mediapipe/calculators/image/image_cropping_calculator.h
+11 -11
View File
@@ -58,24 +58,24 @@ class ImageCroppingCalculator : public CalculatorBase {
ImageCroppingCalculator() = default;
~ImageCroppingCalculator() override = default;
static ::mediapipe::Status GetContract(CalculatorContract* cc);
::mediapipe::Status Open(CalculatorContext* cc) override;
::mediapipe::Status Process(CalculatorContext* cc) override;
::mediapipe::Status Close(CalculatorContext* cc) override;
static mediapipe::Status GetContract(CalculatorContract* cc);
mediapipe::Status Open(CalculatorContext* cc) override;
mediapipe::Status Process(CalculatorContext* cc) override;
mediapipe::Status Close(CalculatorContext* cc) override;
static RectSpec GetCropSpecs(const CalculatorContext* cc, int src_width,
int src_height);
private:
::mediapipe::Status ValidateBorderModeForCPU(CalculatorContext* cc);
::mediapipe::Status ValidateBorderModeForGPU(CalculatorContext* cc);
::mediapipe::Status RenderCpu(CalculatorContext* cc);
::mediapipe::Status RenderGpu(CalculatorContext* cc);
::mediapipe::Status InitGpu(CalculatorContext* cc);
mediapipe::Status ValidateBorderModeForCPU(CalculatorContext* cc);
mediapipe::Status ValidateBorderModeForGPU(CalculatorContext* cc);
mediapipe::Status RenderCpu(CalculatorContext* cc);
mediapipe::Status RenderGpu(CalculatorContext* cc);
mediapipe::Status InitGpu(CalculatorContext* cc);
void GlRender();
void GetOutputDimensions(CalculatorContext* cc, int src_width, int src_height,
int* dst_width, int* dst_height);
::mediapipe::Status GetBorderModeForOpenCV(CalculatorContext* cc,
int* border_mode);
mediapipe::Status GetBorderModeForOpenCV(CalculatorContext* cc,
int* border_mode);
mediapipe::ImageCroppingCalculatorOptions options_;
mediapipe/calculators/image/image_file_properties_calculator.cc
+18 -17
View File
@@ -28,23 +28,24 @@ namespace {
// sqrt(36^2 + 24^2).
static const double SENSOR_DIAGONAL_35MM = std::sqrt(1872.0);
::mediapipe::StatusOr<double> ComputeFocalLengthInPixels(
int image_width, int image_height, double focal_length_35mm,
double focal_length_mm) {
mediapipe::StatusOr<double> ComputeFocalLengthInPixels(int image_width,
int image_height,
double focal_length_35mm,
double focal_length_mm) {
// TODO: Allow returning image file properties even when focal length
// computation is not possible.
if (image_width == 0 || image_height == 0) {
return ::mediapipe::InternalError(
return mediapipe::InternalError(
"Image dimensions should be non-zero to compute focal length in "
"pixels.");
}
if (focal_length_mm == 0) {
return ::mediapipe::InternalError(
return mediapipe::InternalError(
"Focal length in mm should be non-zero to compute focal length in "
"pixels.");
}
if (focal_length_35mm == 0) {
return ::mediapipe::InternalError(
return mediapipe::InternalError(
"Focal length in 35 mm should be non-zero to compute focal length in "
"pixels.");
}
@@ -76,13 +77,13 @@ static const double SENSOR_DIAGONAL_35MM = std::sqrt(1872.0);
return focal_length_pixels;
}
::mediapipe::StatusOr<ImageFileProperties> GetImageFileProperites(
mediapipe::StatusOr<ImageFileProperties> GetImageFileProperites(
const std::string& image_bytes) {
easyexif::EXIFInfo result;
int code = result.parseFrom(image_bytes);
if (code) {
return ::mediapipe::InternalError("Error parsing EXIF, code: " +
std::to_string(code));
return mediapipe::InternalError("Error parsing EXIF, code: " +
std::to_string(code));
}
ImageFileProperties properties;
@@ -125,7 +126,7 @@ static const double SENSOR_DIAGONAL_35MM = std::sqrt(1872.0);
// }
class ImageFilePropertiesCalculator : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
static mediapipe::Status GetContract(CalculatorContract* cc) {
if (cc->Inputs().NumEntries() != 0) {
RET_CHECK(cc->Inputs().NumEntries() == 1);
cc->Inputs().Index(0).Set<std::string>();
@@ -141,10 +142,10 @@ class ImageFilePropertiesCalculator : public CalculatorBase {
cc->OutputSidePackets().Index(0).Set<::mediapipe::ImageFileProperties>();
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status Open(CalculatorContext* cc) override {
mediapipe::Status Open(CalculatorContext* cc) override {
cc->SetOffset(TimestampDiff(0));
if (cc->InputSidePackets().NumEntries() == 1) {
@@ -159,13 +160,13 @@ class ImageFilePropertiesCalculator : public CalculatorBase {
MakePacket<ImageFileProperties>(properties_));
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status Process(CalculatorContext* cc) override {
mediapipe::Status Process(CalculatorContext* cc) override {
if (cc->Inputs().NumEntries() == 1) {
if (cc->Inputs().Index(0).IsEmpty()) {
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
const std::string& image_bytes = cc->Inputs().Index(0).Get<std::string>();
ASSIGN_OR_RETURN(properties_, GetImageFileProperites(image_bytes));
@@ -179,11 +180,11 @@ class ImageFilePropertiesCalculator : public CalculatorBase {
} else {
cc->OutputSidePackets().Index(0).Set(
MakePacket<ImageFileProperties>(properties_)
.At(::mediapipe::Timestamp::Unset()));
.At(mediapipe::Timestamp::Unset()));
}
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
private:
mediapipe/calculators/image/image_properties_calculator.cc
+6 -6
View File
@@ -44,7 +44,7 @@ namespace mediapipe {
// }
class ImagePropertiesCalculator : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
static mediapipe::Status GetContract(CalculatorContract* cc) {
RET_CHECK(cc->Inputs().HasTag(kImageFrameTag) ^
cc->Inputs().HasTag(kGpuBufferTag));
if (cc->Inputs().HasTag(kImageFrameTag)) {
@@ -60,15 +60,15 @@ class ImagePropertiesCalculator : public CalculatorBase {
cc->Outputs().Tag("SIZE").Set<std::pair<int, int>>();
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status Open(CalculatorContext* cc) override {
mediapipe::Status Open(CalculatorContext* cc) override {
cc->SetOffset(TimestampDiff(0));
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status Process(CalculatorContext* cc) override {
mediapipe::Status Process(CalculatorContext* cc) override {
int width;
int height;
@@ -92,7 +92,7 @@ class ImagePropertiesCalculator : public CalculatorBase {
MakePacket<std::pair<int, int>>(width, height)
.At(cc->InputTimestamp()));
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
};
REGISTER_CALCULATOR(ImagePropertiesCalculator);
mediapipe/calculators/image/image_transformation_calculator.cc
+22 -23
View File
@@ -163,16 +163,16 @@ class ImageTransformationCalculator : public CalculatorBase {
ImageTransformationCalculator() = default;
~ImageTransformationCalculator() override = default;
static ::mediapipe::Status GetContract(CalculatorContract* cc);
static mediapipe::Status GetContract(CalculatorContract* cc);
::mediapipe::Status Open(CalculatorContext* cc) override;
::mediapipe::Status Process(CalculatorContext* cc) override;
::mediapipe::Status Close(CalculatorContext* cc) override;
mediapipe::Status Open(CalculatorContext* cc) override;
mediapipe::Status Process(CalculatorContext* cc) override;
mediapipe::Status Close(CalculatorContext* cc) override;
private:
::mediapipe::Status RenderCpu(CalculatorContext* cc);
::mediapipe::Status RenderGpu(CalculatorContext* cc);
::mediapipe::Status GlSetup();
mediapipe::Status RenderCpu(CalculatorContext* cc);
mediapipe::Status RenderGpu(CalculatorContext* cc);
mediapipe::Status GlSetup();
void ComputeOutputDimensions(int input_width, int input_height,
int* output_width, int* output_height);
@@ -199,7 +199,7 @@ class ImageTransformationCalculator : public CalculatorBase {
REGISTER_CALCULATOR(ImageTransformationCalculator);
// static
::mediapipe::Status ImageTransformationCalculator::GetContract(
mediapipe::Status ImageTransformationCalculator::GetContract(
CalculatorContract* cc) {
// Only one input can be set, and the output type must match.
RET_CHECK(cc->Inputs().HasTag(kImageFrameTag) ^
@@ -254,10 +254,10 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
#endif // !MEDIAPIPE_DISABLE_GPU
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status ImageTransformationCalculator::Open(CalculatorContext* cc) {
mediapipe::Status ImageTransformationCalculator::Open(CalculatorContext* cc) {
// Inform the framework that we always output at the same timestamp
// as we receive a packet at.
cc->SetOffset(TimestampDiff(0));
@@ -311,10 +311,10 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
#endif // !MEDIAPIPE_DISABLE_GPU
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status ImageTransformationCalculator::Process(
mediapipe::Status ImageTransformationCalculator::Process(
CalculatorContext* cc) {
// Override values if specified so.
if (cc->Inputs().HasTag("ROTATION_DEGREES") &&
@@ -334,22 +334,21 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
if (use_gpu_) {
#if !defined(MEDIAPIPE_DISABLE_GPU)
if (cc->Inputs().Tag(kGpuBufferTag).IsEmpty()) {
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
return gpu_helper_.RunInGlContext(
[this, cc]() -> ::mediapipe::Status { return RenderGpu(cc); });
[this, cc]() -> mediapipe::Status { return RenderGpu(cc); });
#endif // !MEDIAPIPE_DISABLE_GPU
} else {
if (cc->Inputs().Tag(kImageFrameTag).IsEmpty()) {
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
return RenderCpu(cc);
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status ImageTransformationCalculator::Close(
CalculatorContext* cc) {
mediapipe::Status ImageTransformationCalculator::Close(CalculatorContext* cc) {
if (use_gpu_) {
#if !defined(MEDIAPIPE_DISABLE_GPU)
QuadRenderer* rgb_renderer = rgb_renderer_.release();
@@ -372,10 +371,10 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
#endif // !MEDIAPIPE_DISABLE_GPU
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status ImageTransformationCalculator::RenderCpu(
mediapipe::Status ImageTransformationCalculator::RenderCpu(
CalculatorContext* cc) {
cv::Mat input_mat;
mediapipe::ImageFormat::Format format;
@@ -480,10 +479,10 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
.Tag(kImageFrameTag)
.Add(output_frame.release(), cc->InputTimestamp());
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status ImageTransformationCalculator::RenderGpu(
mediapipe::Status ImageTransformationCalculator::RenderGpu(
CalculatorContext* cc) {
#if !defined(MEDIAPIPE_DISABLE_GPU)
const auto& input = cc->Inputs().Tag(kGpuBufferTag).Get<GpuBuffer>();
@@ -570,7 +569,7 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
#endif // !MEDIAPIPE_DISABLE_GPU
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
void ImageTransformationCalculator::ComputeOutputDimensions(
mediapipe/calculators/image/luminance_calculator.cc
+11 -11
View File
@@ -26,10 +26,10 @@ namespace mediapipe {
// See GlSimpleCalculatorBase for inputs, outputs and input side packets.
class LuminanceCalculator : public GlSimpleCalculator {
public:
::mediapipe::Status GlSetup() override;
::mediapipe::Status GlRender(const GlTexture& src,
const GlTexture& dst) override;
::mediapipe::Status GlTeardown() override;
mediapipe::Status GlSetup() override;
mediapipe::Status GlRender(const GlTexture& src,
const GlTexture& dst) override;
mediapipe::Status GlTeardown() override;
private:
GLuint program_ = 0;
@@ -37,7 +37,7 @@ class LuminanceCalculator : public GlSimpleCalculator {
};
REGISTER_CALCULATOR(LuminanceCalculator);
::mediapipe::Status LuminanceCalculator::GlSetup() {
mediapipe::Status LuminanceCalculator::GlSetup() {
// Load vertex and fragment shaders
const GLint attr_location[NUM_ATTRIBUTES] = {
ATTRIB_VERTEX,
@@ -83,11 +83,11 @@ REGISTER_CALCULATOR(LuminanceCalculator);
(const GLchar**)&attr_name[0], attr_location, &program_);
RET_CHECK(program_) << "Problem initializing the program.";
frame_ = glGetUniformLocation(program_, "video_frame");
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status LuminanceCalculator::GlRender(const GlTexture& src,
const GlTexture& dst) {
mediapipe::Status LuminanceCalculator::GlRender(const GlTexture& src,
const GlTexture& dst) {
static const GLfloat square_vertices[] = {
-1.0f, -1.0f, // bottom left
1.0f, -1.0f, // bottom right
@@ -137,15 +137,15 @@ REGISTER_CALCULATOR(LuminanceCalculator);
glDeleteVertexArrays(1, &vao);
glDeleteBuffers(2, vbo);
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status LuminanceCalculator::GlTeardown() {
mediapipe::Status LuminanceCalculator::GlTeardown() {
if (program_) {
glDeleteProgram(program_);
program_ = 0;
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
} // namespace mediapipe
mediapipe/calculators/image/mask_overlay_calculator.cc
+18 -19
View File
@@ -52,14 +52,14 @@ class MaskOverlayCalculator : public CalculatorBase {
MaskOverlayCalculator() {}
~MaskOverlayCalculator();
static ::mediapipe::Status GetContract(CalculatorContract* cc);
static mediapipe::Status GetContract(CalculatorContract* cc);
::mediapipe::Status Open(CalculatorContext* cc) override;
::mediapipe::Status Process(CalculatorContext* cc) override;
mediapipe::Status Open(CalculatorContext* cc) override;
mediapipe::Status Process(CalculatorContext* cc) override;
::mediapipe::Status GlSetup(
mediapipe::Status GlSetup(
const MaskOverlayCalculatorOptions::MaskChannel mask_channel);
::mediapipe::Status GlRender(const float mask_const);
mediapipe::Status GlRender(const float mask_const);
private:
GlCalculatorHelper helper_;
@@ -73,7 +73,7 @@ class MaskOverlayCalculator : public CalculatorBase {
REGISTER_CALCULATOR(MaskOverlayCalculator);
// static
::mediapipe::Status MaskOverlayCalculator::GetContract(CalculatorContract* cc) {
mediapipe::Status MaskOverlayCalculator::GetContract(CalculatorContract* cc) {
MP_RETURN_IF_ERROR(GlCalculatorHelper::UpdateContract(cc));
cc->Inputs().Get("VIDEO", 0).Set<GpuBuffer>();
cc->Inputs().Get("VIDEO", 1).Set<GpuBuffer>();
@@ -82,14 +82,13 @@ REGISTER_CALCULATOR(MaskOverlayCalculator);
else if (cc->Inputs().HasTag("CONST_MASK"))
cc->Inputs().Tag("CONST_MASK").Set<float>();
else
return ::mediapipe::Status(
::mediapipe::StatusCode::kNotFound,
"At least one mask input stream must be present.");
return mediapipe::Status(mediapipe::StatusCode::kNotFound,
"At least one mask input stream must be present.");
cc->Outputs().Tag("OUTPUT").Set<GpuBuffer>();
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status MaskOverlayCalculator::Open(CalculatorContext* cc) {
mediapipe::Status MaskOverlayCalculator::Open(CalculatorContext* cc) {
cc->SetOffset(TimestampDiff(0));
if (cc->Inputs().HasTag("MASK")) {
use_mask_tex_ = true;
@@ -97,8 +96,8 @@ REGISTER_CALCULATOR(MaskOverlayCalculator);
return helper_.Open(cc);
}
::mediapipe::Status MaskOverlayCalculator::Process(CalculatorContext* cc) {
return helper_.RunInGlContext([this, &cc]() -> ::mediapipe::Status {
mediapipe::Status MaskOverlayCalculator::Process(CalculatorContext* cc) {
return helper_.RunInGlContext([this, &cc]() -> mediapipe::Status {
if (!initialized_) {
const auto& options = cc->Options<MaskOverlayCalculatorOptions>();
const auto mask_channel = options.mask_channel();
@@ -116,7 +115,7 @@ REGISTER_CALCULATOR(MaskOverlayCalculator);
if (mask_packet.IsEmpty()) {
cc->Outputs().Tag("OUTPUT").AddPacket(input1_packet);
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
const auto& input0_buffer = cc->Inputs().Get("VIDEO", 0).Get<GpuBuffer>();
@@ -173,11 +172,11 @@ REGISTER_CALCULATOR(MaskOverlayCalculator);
dst.Release();
cc->Outputs().Tag("OUTPUT").Add(output.release(), cc->InputTimestamp());
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
});
}
::mediapipe::Status MaskOverlayCalculator::GlSetup(
mediapipe::Status MaskOverlayCalculator::GlSetup(
const MaskOverlayCalculatorOptions::MaskChannel mask_channel) {
// Load vertex and fragment shaders
const GLint attr_location[NUM_ATTRIBUTES] = {
@@ -248,10 +247,10 @@ REGISTER_CALCULATOR(MaskOverlayCalculator);
unif_frame1_ = glGetUniformLocation(program_, "frame1");
unif_frame2_ = glGetUniformLocation(program_, "frame2");
unif_mask_ = glGetUniformLocation(program_, "mask");
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status MaskOverlayCalculator::GlRender(const float mask_const) {
mediapipe::Status MaskOverlayCalculator::GlRender(const float mask_const) {
glUseProgram(program_);
glVertexAttribPointer(ATTRIB_VERTEX, 2, GL_FLOAT, 0, 0, kBasicSquareVertices);
glEnableVertexAttribArray(ATTRIB_VERTEX);
@@ -267,7 +266,7 @@ REGISTER_CALCULATOR(MaskOverlayCalculator);
glUniform1f(unif_mask_, mask_const);
glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
MaskOverlayCalculator::~MaskOverlayCalculator() {
mediapipe/calculators/image/opencv_encoded_image_to_image_frame_calculator.cc
+11 -11
View File
@@ -34,29 +34,29 @@ namespace mediapipe {
// }
class OpenCvEncodedImageToImageFrameCalculator : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc);
::mediapipe::Status Open(CalculatorContext* cc) override;
::mediapipe::Status Process(CalculatorContext* cc) override;
static mediapipe::Status GetContract(CalculatorContract* cc);
mediapipe::Status Open(CalculatorContext* cc) override;
mediapipe::Status Process(CalculatorContext* cc) override;
private:
mediapipe::OpenCvEncodedImageToImageFrameCalculatorOptions options_;
};
::mediapipe::Status OpenCvEncodedImageToImageFrameCalculator::GetContract(
mediapipe::Status OpenCvEncodedImageToImageFrameCalculator::GetContract(
CalculatorContract* cc) {
cc->Inputs().Index(0).Set<std::string>();
cc->Outputs().Index(0).Set<ImageFrame>();
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status OpenCvEncodedImageToImageFrameCalculator::Open(
mediapipe::Status OpenCvEncodedImageToImageFrameCalculator::Open(
CalculatorContext* cc) {
options_ =
cc->Options<mediapipe::OpenCvEncodedImageToImageFrameCalculatorOptions>();
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status OpenCvEncodedImageToImageFrameCalculator::Process(
mediapipe::Status OpenCvEncodedImageToImageFrameCalculator::Process(
CalculatorContext* cc) {
const std::string& contents = cc->Inputs().Index(0).Get<std::string>();
const std::vector<char> contents_vector(contents.begin(), contents.end());
@@ -84,10 +84,10 @@ class OpenCvEncodedImageToImageFrameCalculator : public CalculatorBase {
cv::cvtColor(decoded_mat, output_mat, cv::COLOR_BGR2RGB);
break;
case 4:
return ::mediapipe::UnimplementedErrorBuilder(MEDIAPIPE_LOC)
return mediapipe::UnimplementedErrorBuilder(MEDIAPIPE_LOC)
<< "4-channel image isn't supported yet";
default:
return ::mediapipe::FailedPreconditionErrorBuilder(MEDIAPIPE_LOC)
return mediapipe::FailedPreconditionErrorBuilder(MEDIAPIPE_LOC)
<< "Unsupported number of channels: " << decoded_mat.channels();
}
std::unique_ptr<ImageFrame> output_frame = absl::make_unique<ImageFrame>(
@@ -95,7 +95,7 @@ class OpenCvEncodedImageToImageFrameCalculator : public CalculatorBase {
ImageFrame::kGlDefaultAlignmentBoundary);
output_mat.copyTo(formats::MatView(output_frame.get()));
cc->Outputs().Index(0).Add(output_frame.release(), cc->InputTimestamp());
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
REGISTER_CALCULATOR(OpenCvEncodedImageToImageFrameCalculator);
mediapipe/calculators/image/opencv_image_encoder_calculator.cc
+15 -16
View File
@@ -38,30 +38,29 @@ namespace mediapipe {
// }
class OpenCvImageEncoderCalculator : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc);
::mediapipe::Status Open(CalculatorContext* cc) override;
::mediapipe::Status Process(CalculatorContext* cc) override;
::mediapipe::Status Close(CalculatorContext* cc) override;
static mediapipe::Status GetContract(CalculatorContract* cc);
mediapipe::Status Open(CalculatorContext* cc) override;
mediapipe::Status Process(CalculatorContext* cc) override;
mediapipe::Status Close(CalculatorContext* cc) override;
private:
int encoding_quality_;
};
::mediapipe::Status OpenCvImageEncoderCalculator::GetContract(
mediapipe::Status OpenCvImageEncoderCalculator::GetContract(
CalculatorContract* cc) {
cc->Inputs().Index(0).Set<ImageFrame>();
cc->Outputs().Index(0).Set<OpenCvImageEncoderCalculatorResults>();
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status OpenCvImageEncoderCalculator::Open(CalculatorContext* cc) {
mediapipe::Status OpenCvImageEncoderCalculator::Open(CalculatorContext* cc) {
auto options = cc->Options<OpenCvImageEncoderCalculatorOptions>();
encoding_quality_ = options.quality();
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status OpenCvImageEncoderCalculator::Process(
CalculatorContext* cc) {
mediapipe::Status OpenCvImageEncoderCalculator::Process(CalculatorContext* cc) {
const ImageFrame& image_frame = cc->Inputs().Index(0).Get<ImageFrame>();
CHECK_EQ(1, image_frame.ByteDepth());
@@ -85,10 +84,10 @@ class OpenCvImageEncoderCalculator : public CalculatorBase {
encoded_result->set_colorspace(OpenCvImageEncoderCalculatorResults::RGB);
break;
case 4:
return ::mediapipe::UnimplementedErrorBuilder(MEDIAPIPE_LOC)
return mediapipe::UnimplementedErrorBuilder(MEDIAPIPE_LOC)
<< "4-channel image isn't supported yet";
default:
return ::mediapipe::FailedPreconditionErrorBuilder(MEDIAPIPE_LOC)
return mediapipe::FailedPreconditionErrorBuilder(MEDIAPIPE_LOC)
<< "Unsupported number of channels: " << original_mat.channels();
}
@@ -101,7 +100,7 @@ class OpenCvImageEncoderCalculator : public CalculatorBase {
// Check its JpegEncoder::write() in "imgcodecs/src/grfmt_jpeg.cpp" for more
// info.
if (!cv::imencode(".jpg", input_mat, encode_buffer, parameters)) {
return ::mediapipe::InternalErrorBuilder(MEDIAPIPE_LOC)
return mediapipe::InternalErrorBuilder(MEDIAPIPE_LOC)
<< "Fail to encode the image to be jpeg format.";
}
@@ -109,11 +108,11 @@ class OpenCvImageEncoderCalculator : public CalculatorBase {
reinterpret_cast<const char*>(&encode_buffer[0]), encode_buffer.size())));
cc->Outputs().Index(0).Add(encoded_result.release(), cc->InputTimestamp());
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status OpenCvImageEncoderCalculator::Close(CalculatorContext* cc) {
return ::mediapipe::OkStatus();
mediapipe::Status OpenCvImageEncoderCalculator::Close(CalculatorContext* cc) {
return mediapipe::OkStatus();
}
REGISTER_CALCULATOR(OpenCvImageEncoderCalculator);
mediapipe/calculators/image/opencv_put_text_calculator.cc
+6 -7
View File
@@ -32,18 +32,17 @@ namespace mediapipe {
// TODO: Generalize the calculator for other text use cases.
class OpenCvPutTextCalculator : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc);
::mediapipe::Status Process(CalculatorContext* cc) override;
static mediapipe::Status GetContract(CalculatorContract* cc);
mediapipe::Status Process(CalculatorContext* cc) override;
};
::mediapipe::Status OpenCvPutTextCalculator::GetContract(
CalculatorContract* cc) {
mediapipe::Status OpenCvPutTextCalculator::GetContract(CalculatorContract* cc) {
cc->Inputs().Index(0).Set<std::string>();
cc->Outputs().Index(0).Set<ImageFrame>();
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status OpenCvPutTextCalculator::Process(CalculatorContext* cc) {
mediapipe::Status OpenCvPutTextCalculator::Process(CalculatorContext* cc) {
const std::string& text_content = cc->Inputs().Index(0).Get<std::string>();
cv::Mat mat = cv::Mat::zeros(640, 640, CV_8UC4);
cv::putText(mat, text_content, cv::Point(15, 70), cv::FONT_HERSHEY_PLAIN, 3,
@@ -52,7 +51,7 @@ class OpenCvPutTextCalculator : public CalculatorBase {
ImageFormat::SRGBA, mat.size().width, mat.size().height);
mat.copyTo(formats::MatView(output_frame.get()));
cc->Outputs().Index(0).Add(output_frame.release(), cc->InputTimestamp());
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
REGISTER_CALCULATOR(OpenCvPutTextCalculator);
mediapipe/calculators/image/recolor_calculator.cc
+28 -28
View File
@@ -84,17 +84,17 @@ class RecolorCalculator : public CalculatorBase {
RecolorCalculator() = default;
~RecolorCalculator() override = default;
static ::mediapipe::Status GetContract(CalculatorContract* cc);
static mediapipe::Status GetContract(CalculatorContract* cc);
::mediapipe::Status Open(CalculatorContext* cc) override;
::mediapipe::Status Process(CalculatorContext* cc) override;
::mediapipe::Status Close(CalculatorContext* cc) override;
mediapipe::Status Open(CalculatorContext* cc) override;
mediapipe::Status Process(CalculatorContext* cc) override;
mediapipe::Status Close(CalculatorContext* cc) override;
private:
::mediapipe::Status LoadOptions(CalculatorContext* cc);
::mediapipe::Status InitGpu(CalculatorContext* cc);
::mediapipe::Status RenderGpu(CalculatorContext* cc);
::mediapipe::Status RenderCpu(CalculatorContext* cc);
mediapipe::Status LoadOptions(CalculatorContext* cc);
mediapipe::Status InitGpu(CalculatorContext* cc);
mediapipe::Status RenderGpu(CalculatorContext* cc);
mediapipe::Status RenderCpu(CalculatorContext* cc);
void GlRender();
bool initialized_ = false;
@@ -110,7 +110,7 @@ class RecolorCalculator : public CalculatorBase {
REGISTER_CALCULATOR(RecolorCalculator);
// static
::mediapipe::Status RecolorCalculator::GetContract(CalculatorContract* cc) {
mediapipe::Status RecolorCalculator::GetContract(CalculatorContract* cc) {
RET_CHECK(!cc->Inputs().GetTags().empty());
RET_CHECK(!cc->Outputs().GetTags().empty());
@@ -159,10 +159,10 @@ REGISTER_CALCULATOR(RecolorCalculator);
#endif // !MEDIAPIPE_DISABLE_GPU
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status RecolorCalculator::Open(CalculatorContext* cc) {
mediapipe::Status RecolorCalculator::Open(CalculatorContext* cc) {
cc->SetOffset(TimestampDiff(0));
if (cc->Inputs().HasTag(kGpuBufferTag)) {
@@ -174,29 +174,29 @@ REGISTER_CALCULATOR(RecolorCalculator);
MP_RETURN_IF_ERROR(LoadOptions(cc));
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status RecolorCalculator::Process(CalculatorContext* cc) {
mediapipe::Status RecolorCalculator::Process(CalculatorContext* cc) {
if (use_gpu_) {
#if !defined(MEDIAPIPE_DISABLE_GPU)
MP_RETURN_IF_ERROR(
gpu_helper_.RunInGlContext([this, &cc]() -> ::mediapipe::Status {
gpu_helper_.RunInGlContext([this, &cc]() -> mediapipe::Status {
if (!initialized_) {
MP_RETURN_IF_ERROR(InitGpu(cc));
initialized_ = true;
}
MP_RETURN_IF_ERROR(RenderGpu(cc));
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}));
#endif // !MEDIAPIPE_DISABLE_GPU
} else {
MP_RETURN_IF_ERROR(RenderCpu(cc));
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status RecolorCalculator::Close(CalculatorContext* cc) {
mediapipe::Status RecolorCalculator::Close(CalculatorContext* cc) {
#if !defined(MEDIAPIPE_DISABLE_GPU)
gpu_helper_.RunInGlContext([this] {
if (program_) glDeleteProgram(program_);
@@ -204,12 +204,12 @@ REGISTER_CALCULATOR(RecolorCalculator);
});
#endif // !MEDIAPIPE_DISABLE_GPU
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status RecolorCalculator::RenderCpu(CalculatorContext* cc) {
mediapipe::Status RecolorCalculator::RenderCpu(CalculatorContext* cc) {
if (cc->Inputs().Tag(kMaskCpuTag).IsEmpty()) {
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
// Get inputs and setup output.
const auto& input_img = cc->Inputs().Tag(kImageFrameTag).Get<ImageFrame>();
@@ -265,12 +265,12 @@ REGISTER_CALCULATOR(RecolorCalculator);
.Tag(kImageFrameTag)
.Add(output_img.release(), cc->InputTimestamp());
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status RecolorCalculator::RenderGpu(CalculatorContext* cc) {
mediapipe::Status RecolorCalculator::RenderGpu(CalculatorContext* cc) {
if (cc->Inputs().Tag(kMaskGpuTag).IsEmpty()) {
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
#if !defined(MEDIAPIPE_DISABLE_GPU)
// Get inputs and setup output.
@@ -313,7 +313,7 @@ REGISTER_CALCULATOR(RecolorCalculator);
dst_tex.Release();
#endif // !MEDIAPIPE_DISABLE_GPU
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
void RecolorCalculator::GlRender() {
@@ -368,7 +368,7 @@ void RecolorCalculator::GlRender() {
#endif // !MEDIAPIPE_DISABLE_GPU
}
::mediapipe::Status RecolorCalculator::LoadOptions(CalculatorContext* cc) {
mediapipe::Status RecolorCalculator::LoadOptions(CalculatorContext* cc) {
const auto& options = cc->Options<mediapipe::RecolorCalculatorOptions>();
mask_channel_ = options.mask_channel();
@@ -379,10 +379,10 @@ void RecolorCalculator::GlRender() {
color_.push_back(options.color().g());
color_.push_back(options.color().b());
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status RecolorCalculator::InitGpu(CalculatorContext* cc) {
mediapipe::Status RecolorCalculator::InitGpu(CalculatorContext* cc) {
#if !defined(MEDIAPIPE_DISABLE_GPU)
const GLint attr_location[NUM_ATTRIBUTES] = {
ATTRIB_VERTEX,
@@ -454,7 +454,7 @@ void RecolorCalculator::GlRender() {
color_[1] / 255.0, color_[2] / 255.0);
#endif // !MEDIAPIPE_DISABLE_GPU
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
} // namespace mediapipe
mediapipe/calculators/image/scale_image_calculator.cc
+32 -32
View File
@@ -44,7 +44,7 @@ namespace {
// Given an upscaling algorithm, determine which OpenCV interpolation algorithm
// to use.
::mediapipe::Status FindInterpolationAlgorithm(
mediapipe::Status FindInterpolationAlgorithm(
ScaleImageCalculatorOptions::ScaleAlgorithm upscaling_algorithm,
int* interpolation_algorithm) {
switch (upscaling_algorithm) {
@@ -70,7 +70,7 @@ namespace {
RET_CHECK_FAIL() << absl::Substitute("Unknown upscaling algorithm: $0",
upscaling_algorithm);
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
void CropImageFrame(const ImageFrame& original, int col_start, int row_start,
@@ -147,7 +147,7 @@ class ScaleImageCalculator : public CalculatorBase {
ScaleImageCalculator();
~ScaleImageCalculator() override;
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
static mediapipe::Status GetContract(CalculatorContract* cc) {
ScaleImageCalculatorOptions options =
cc->Options<ScaleImageCalculatorOptions>();
@@ -184,35 +184,35 @@ class ScaleImageCalculator : public CalculatorBase {
if (cc->Inputs().HasTag("OVERRIDE_OPTIONS")) {
cc->Inputs().Tag("OVERRIDE_OPTIONS").Set<ScaleImageCalculatorOptions>();
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
// From Calculator.
::mediapipe::Status Open(CalculatorContext* cc) override;
::mediapipe::Status Process(CalculatorContext* cc) override;
mediapipe::Status Open(CalculatorContext* cc) override;
mediapipe::Status Process(CalculatorContext* cc) override;
private:
// Initialize some data members from options_. This can be called either from
// Open or Process depending on whether OVERRIDE_OPTIONS is used.
::mediapipe::Status InitializeFromOptions();
mediapipe::Status InitializeFromOptions();
// Initialize crop and output parameters based on set member variable
// values. This function will also send the header information on
// the VIDEO_HEADER stream if it hasn't been done yet.
::mediapipe::Status InitializeFrameInfo(CalculatorContext* cc);
mediapipe::Status InitializeFrameInfo(CalculatorContext* cc);
// Validate that input_format_ and output_format_ are supported image
// formats.
::mediapipe::Status ValidateImageFormats() const;
mediapipe::Status ValidateImageFormats() const;
// Validate that the image frame has the proper format and dimensions.
// If the dimensions and format weren't initialized by the header,
// then the first frame on which this function is called is used
// to initialize.
::mediapipe::Status ValidateImageFrame(CalculatorContext* cc,
const ImageFrame& image_frame);
mediapipe::Status ValidateImageFrame(CalculatorContext* cc,
const ImageFrame& image_frame);
// Validate that the YUV image has the proper dimensions. If the
// dimensions weren't initialized by the header, then the first image
// on which this function is called is used to initialize.
::mediapipe::Status ValidateYUVImage(CalculatorContext* cc,
const YUVImage& yuv_image);
mediapipe::Status ValidateYUVImage(CalculatorContext* cc,
const YUVImage& yuv_image);
bool has_header_; // True if the input stream has a header.
int input_width_;
@@ -251,7 +251,7 @@ ScaleImageCalculator::ScaleImageCalculator() {}
ScaleImageCalculator::~ScaleImageCalculator() {}
::mediapipe::Status ScaleImageCalculator::InitializeFrameInfo(
mediapipe::Status ScaleImageCalculator::InitializeFrameInfo(
CalculatorContext* cc) {
MP_RETURN_IF_ERROR(
scale_image::FindCropDimensions(input_width_, input_height_, //
@@ -299,10 +299,10 @@ ScaleImageCalculator::~ScaleImageCalculator() {}
.Add(header.release(), Timestamp::PreStream());
cc->Outputs().Tag("VIDEO_HEADER").Close();
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status ScaleImageCalculator::Open(CalculatorContext* cc) {
mediapipe::Status ScaleImageCalculator::Open(CalculatorContext* cc) {
options_ = cc->Options<ScaleImageCalculatorOptions>();
input_data_id_ = cc->Inputs().GetId("FRAMES", 0);
@@ -339,7 +339,7 @@ ScaleImageCalculator::~ScaleImageCalculator() {}
// has a header. At this point in the code, the ScaleImageCalculator
// config may be changed by the new options at PreStream, so the output
// header can't be determined.
return ::mediapipe::InvalidArgumentError(
return mediapipe::InvalidArgumentError(
"OVERRIDE_OPTIONS stream can't be used when the main input stream "
"has a header.");
}
@@ -406,10 +406,10 @@ ScaleImageCalculator::~ScaleImageCalculator() {}
}
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status ScaleImageCalculator::InitializeFromOptions() {
mediapipe::Status ScaleImageCalculator::InitializeFromOptions() {
if (options_.has_input_format()) {
input_format_ = options_.input_format();
} else {
@@ -423,10 +423,10 @@ ScaleImageCalculator::~ScaleImageCalculator() {}
downscaler_.reset(new ImageResizer(options_.post_sharpening_coefficient()));
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status ScaleImageCalculator::ValidateImageFormats() const {
mediapipe::Status ScaleImageCalculator::ValidateImageFormats() const {
RET_CHECK_NE(input_format_, ImageFormat::UNKNOWN)
<< "The input image format was UNKNOWN.";
RET_CHECK_NE(output_format_, ImageFormat::UNKNOWN)
@@ -440,10 +440,10 @@ ScaleImageCalculator::~ScaleImageCalculator() {}
input_format_ == ImageFormat::YCBCR420P)
<< "Conversion of the color space (except from "
"YCbCr420P to SRGB) is not yet supported.";
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status ScaleImageCalculator::ValidateImageFrame(
mediapipe::Status ScaleImageCalculator::ValidateImageFrame(
CalculatorContext* cc, const ImageFrame& image_frame) {
if (!has_header_) {
if (input_width_ != image_frame.Width() ||
@@ -494,10 +494,10 @@ ScaleImageCalculator::~ScaleImageCalculator() {}
image_frame_format_desc, " but expected ", input_format_desc));
}
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status ScaleImageCalculator::ValidateYUVImage(
mediapipe::Status ScaleImageCalculator::ValidateYUVImage(
CalculatorContext* cc, const YUVImage& yuv_image) {
CHECK_EQ(input_format_, ImageFormat::YCBCR420P);
if (!has_header_) {
@@ -528,14 +528,14 @@ ScaleImageCalculator::~ScaleImageCalculator() {}
input_width_, "x", input_height_));
}
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status ScaleImageCalculator::Process(CalculatorContext* cc) {
mediapipe::Status ScaleImageCalculator::Process(CalculatorContext* cc) {
if (cc->InputTimestamp() == Timestamp::PreStream()) {
if (cc->Inputs().HasTag("OVERRIDE_OPTIONS")) {
if (cc->Inputs().Tag("OVERRIDE_OPTIONS").IsEmpty()) {
return ::mediapipe::InvalidArgumentError(
return mediapipe::InvalidArgumentError(
"The OVERRIDE_OPTIONS input stream must be non-empty at PreStream "
"time if used.");
}
@@ -549,7 +549,7 @@ ScaleImageCalculator::~ScaleImageCalculator() {}
input_video_header_ = cc->Inputs().Tag("VIDEO_HEADER").Get<VideoHeader>();
}
if (cc->Inputs().Get(input_data_id_).IsEmpty()) {
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
}
@@ -603,7 +603,7 @@ ScaleImageCalculator::~ScaleImageCalculator() {}
cc->Outputs()
.Get(output_data_id_)
.Add(output_image.release(), cc->InputTimestamp());
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
} else {
image_frame = &cc->Inputs().Get(input_data_id_).Get<ImageFrame>();
@@ -664,7 +664,7 @@ ScaleImageCalculator::~ScaleImageCalculator() {}
.Add(output_frame.release(), cc->InputTimestamp());
}
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
// Rescale the image frame.
@@ -698,7 +698,7 @@ ScaleImageCalculator::~ScaleImageCalculator() {}
cc->Outputs()
.Get(output_data_id_)
.Add(output_frame.release(), cc->InputTimestamp());
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
} // namespace mediapipe
mediapipe/calculators/image/scale_image_utils.cc
+16 -17
View File
@@ -35,11 +35,11 @@ double ParseRational(const std::string& rational) {
}
} // namespace
::mediapipe::Status FindCropDimensions(int input_width, int input_height, //
const std::string& min_aspect_ratio, //
const std::string& max_aspect_ratio, //
int* crop_width, int* crop_height, //
int* col_start, int* row_start) {
mediapipe::Status FindCropDimensions(int input_width, int input_height, //
const std::string& min_aspect_ratio, //
const std::string& max_aspect_ratio, //
int* crop_width, int* crop_height, //
int* col_start, int* row_start) {
CHECK(crop_width);
CHECK(crop_height);
CHECK(col_start);
@@ -85,17 +85,16 @@ double ParseRational(const std::string& rational) {
CHECK_LE(*crop_width, input_width);
CHECK_LE(*crop_height, input_height);
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status FindOutputDimensions(int input_width, //
int input_height, //
int target_width, //
int target_height, //
bool preserve_aspect_ratio, //
int scale_to_multiple_of, //
int* output_width,
int* output_height) {
mediapipe::Status FindOutputDimensions(int input_width, //
int input_height, //
int target_width, //
int target_height, //
bool preserve_aspect_ratio, //
int scale_to_multiple_of, //
int* output_width, int* output_height) {
CHECK(output_width);
CHECK(output_height);
@@ -123,7 +122,7 @@ double ParseRational(const std::string& rational) {
*output_width = target_width;
*output_height = target_height;
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
if (target_width > 0) {
@@ -140,7 +139,7 @@ double ParseRational(const std::string& rational) {
// was within the image, so use these dimensions.
*output_width = try_width;
*output_height = try_height;
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
}
@@ -158,7 +157,7 @@ double ParseRational(const std::string& rational) {
// was within the image, so use these dimensions.
*output_width = try_width;
*output_height = try_height;
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
}
RET_CHECK_FAIL()
mediapipe/calculators/image/scale_image_utils.h
+11 -11
View File
@@ -28,11 +28,11 @@ namespace scale_image {
// is a centered, cropped portion of the image that falls within the min
// and max aspect ratio. If either the min or max aspect ratio argument
// is empty or has a 0 in the numerator or denominator then it is ignored.
::mediapipe::Status FindCropDimensions(int input_width, int input_height, //
const std::string& min_aspect_ratio, //
const std::string& max_aspect_ratio, //
int* crop_width, int* crop_height, //
int* col_start, int* row_start);
mediapipe::Status FindCropDimensions(int input_width, int input_height, //
const std::string& min_aspect_ratio, //
const std::string& max_aspect_ratio, //
int* crop_width, int* crop_height, //
int* col_start, int* row_start);
// Given an input width and height, a target width and height, whether to
// preserve the aspect ratio, and whether to round-down to the multiple of a
@@ -43,12 +43,12 @@ namespace scale_image {
// output_height will be reduced as necessary to preserve_aspect_ratio if the
// option is specified. If preserving the aspect ratio is desired, you must set
// scale_to_multiple_of to 2.
::mediapipe::Status FindOutputDimensions(int input_width, int input_height, //
int target_width,
int target_height, //
bool preserve_aspect_ratio, //
int scale_to_multiple_of, //
int* output_width, int* output_height);
mediapipe::Status FindOutputDimensions(int input_width, int input_height, //
int target_width,
int target_height, //
bool preserve_aspect_ratio, //
int scale_to_multiple_of, //
int* output_width, int* output_height);
} // namespace scale_image
} // namespace mediapipe
mediapipe/calculators/image/set_alpha_calculator.cc
+27 -27
View File
@@ -87,18 +87,18 @@ class SetAlphaCalculator : public CalculatorBase {
SetAlphaCalculator() = default;
~SetAlphaCalculator() override = default;
static ::mediapipe::Status GetContract(CalculatorContract* cc);
static mediapipe::Status GetContract(CalculatorContract* cc);
// From Calculator.
::mediapipe::Status Open(CalculatorContext* cc) override;
::mediapipe::Status Process(CalculatorContext* cc) override;
::mediapipe::Status Close(CalculatorContext* cc) override;
mediapipe::Status Open(CalculatorContext* cc) override;
mediapipe::Status Process(CalculatorContext* cc) override;
mediapipe::Status Close(CalculatorContext* cc) override;
private:
::mediapipe::Status RenderGpu(CalculatorContext* cc);
::mediapipe::Status RenderCpu(CalculatorContext* cc);
mediapipe::Status RenderGpu(CalculatorContext* cc);
mediapipe::Status RenderCpu(CalculatorContext* cc);
::mediapipe::Status GlSetup(CalculatorContext* cc);
mediapipe::Status GlSetup(CalculatorContext* cc);
void GlRender(CalculatorContext* cc);
mediapipe::SetAlphaCalculatorOptions options_;
@@ -113,18 +113,18 @@ class SetAlphaCalculator : public CalculatorBase {
};
REGISTER_CALCULATOR(SetAlphaCalculator);
::mediapipe::Status SetAlphaCalculator::GetContract(CalculatorContract* cc) {
mediapipe::Status SetAlphaCalculator::GetContract(CalculatorContract* cc) {
CHECK_GE(cc->Inputs().NumEntries(), 1);
bool use_gpu = false;
if (cc->Inputs().HasTag(kInputFrameTag) &&
cc->Inputs().HasTag(kInputFrameTagGpu)) {
return ::mediapipe::InternalError("Cannot have multiple input images.");
return mediapipe::InternalError("Cannot have multiple input images.");
}
if (cc->Inputs().HasTag(kInputFrameTagGpu) !=
cc->Outputs().HasTag(kOutputFrameTagGpu)) {
return ::mediapipe::InternalError("GPU output must have GPU input.");
return mediapipe::InternalError("GPU output must have GPU input.");
}
// Input image to add/edit alpha channel.
@@ -166,10 +166,10 @@ REGISTER_CALCULATOR(SetAlphaCalculator);
#endif // !MEDIAPIPE_DISABLE_GPU
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status SetAlphaCalculator::Open(CalculatorContext* cc) {
mediapipe::Status SetAlphaCalculator::Open(CalculatorContext* cc) {
cc->SetOffset(TimestampDiff(0));
options_ = cc->Options<mediapipe::SetAlphaCalculatorOptions>();
@@ -198,30 +198,30 @@ REGISTER_CALCULATOR(SetAlphaCalculator);
#endif
} // !MEDIAPIPE_DISABLE_GPU
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status SetAlphaCalculator::Process(CalculatorContext* cc) {
mediapipe::Status SetAlphaCalculator::Process(CalculatorContext* cc) {
if (use_gpu_) {
#if !defined(MEDIAPIPE_DISABLE_GPU)
MP_RETURN_IF_ERROR(
gpu_helper_.RunInGlContext([this, cc]() -> ::mediapipe::Status {
gpu_helper_.RunInGlContext([this, cc]() -> mediapipe::Status {
if (!gpu_initialized_) {
MP_RETURN_IF_ERROR(GlSetup(cc));
gpu_initialized_ = true;
}
MP_RETURN_IF_ERROR(RenderGpu(cc));
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}));
#endif // !MEDIAPIPE_DISABLE_GPU
} else {
MP_RETURN_IF_ERROR(RenderCpu(cc));
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status SetAlphaCalculator::Close(CalculatorContext* cc) {
mediapipe::Status SetAlphaCalculator::Close(CalculatorContext* cc) {
#if !defined(MEDIAPIPE_DISABLE_GPU)
gpu_helper_.RunInGlContext([this] {
if (program_) glDeleteProgram(program_);
@@ -229,12 +229,12 @@ REGISTER_CALCULATOR(SetAlphaCalculator);
});
#endif // !MEDIAPIPE_DISABLE_GPU
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status SetAlphaCalculator::RenderCpu(CalculatorContext* cc) {
mediapipe::Status SetAlphaCalculator::RenderCpu(CalculatorContext* cc) {
if (cc->Inputs().Tag(kInputFrameTag).IsEmpty()) {
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
// Setup source image
@@ -294,12 +294,12 @@ REGISTER_CALCULATOR(SetAlphaCalculator);
.Tag(kOutputFrameTag)
.Add(output_frame.release(), cc->InputTimestamp());
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status SetAlphaCalculator::RenderGpu(CalculatorContext* cc) {
mediapipe::Status SetAlphaCalculator::RenderGpu(CalculatorContext* cc) {
if (cc->Inputs().Tag(kInputFrameTagGpu).IsEmpty()) {
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
#if !defined(MEDIAPIPE_DISABLE_GPU)
// Setup source texture.
@@ -356,7 +356,7 @@ REGISTER_CALCULATOR(SetAlphaCalculator);
output_texture.Release();
#endif // !MEDIAPIPE_DISABLE_GPU
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
void SetAlphaCalculator::GlRender(CalculatorContext* cc) {
@@ -412,7 +412,7 @@ void SetAlphaCalculator::GlRender(CalculatorContext* cc) {
#endif // !MEDIAPIPE_DISABLE_GPU
}
::mediapipe::Status SetAlphaCalculator::GlSetup(CalculatorContext* cc) {
mediapipe::Status SetAlphaCalculator::GlSetup(CalculatorContext* cc) {
#if !defined(MEDIAPIPE_DISABLE_GPU)
const GLint attr_location[NUM_ATTRIBUTES] = {
ATTRIB_VERTEX,
@@ -468,7 +468,7 @@ void SetAlphaCalculator::GlRender(CalculatorContext* cc) {
#endif // !MEDIAPIPE_DISABLE_GPU
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
} // namespace mediapipe
mediapipe/calculators/image/sobel_edges_calculator.cc
+11 -11
View File
@@ -27,10 +27,10 @@ namespace mediapipe {
// See GlSimpleCalculatorBase for inputs, outputs and input side packets.
class SobelEdgesCalculator : public GlSimpleCalculator {
public:
::mediapipe::Status GlSetup() override;
::mediapipe::Status GlRender(const GlTexture& src,
const GlTexture& dst) override;
::mediapipe::Status GlTeardown() override;
mediapipe::Status GlSetup() override;
mediapipe::Status GlRender(const GlTexture& src,
const GlTexture& dst) override;
mediapipe::Status GlTeardown() override;
private:
GLuint program_ = 0;
@@ -40,7 +40,7 @@ class SobelEdgesCalculator : public GlSimpleCalculator {
};
REGISTER_CALCULATOR(SobelEdgesCalculator);
::mediapipe::Status SobelEdgesCalculator::GlSetup() {
mediapipe::Status SobelEdgesCalculator::GlSetup() {
// Load vertex and fragment shaders
const GLint attr_location[NUM_ATTRIBUTES] = {
ATTRIB_VERTEX,
@@ -166,11 +166,11 @@ REGISTER_CALCULATOR(SobelEdgesCalculator);
frame_ = glGetUniformLocation(program_, "inputImage");
pixel_w_ = glGetUniformLocation(program_, "pixelW");
pixel_h_ = glGetUniformLocation(program_, "pixelH");
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status SobelEdgesCalculator::GlRender(const GlTexture& src,
const GlTexture& dst) {
mediapipe::Status SobelEdgesCalculator::GlRender(const GlTexture& src,
const GlTexture& dst) {
static const GLfloat square_vertices[] = {
-1.0f, -1.0f, // bottom left
1.0f, -1.0f, // bottom right
@@ -225,15 +225,15 @@ REGISTER_CALCULATOR(SobelEdgesCalculator);
glDeleteVertexArrays(1, &vao);
glDeleteBuffers(2, vbo);
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status SobelEdgesCalculator::GlTeardown() {
mediapipe::Status SobelEdgesCalculator::GlTeardown() {
if (program_) {
glDeleteProgram(program_);
program_ = 0;
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
} // namespace mediapipe
mediapipe/calculators/internal/callback_packet_calculator.cc
+9 -9
View File
@@ -50,7 +50,7 @@ void DumpPostStreamPacket(Packet* post_stream_packet, const Packet& packet) {
// while that pointer is still alive.
class CallbackPacketCalculator : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
static mediapipe::Status GetContract(CalculatorContract* cc) {
const auto& options = cc->Options<CallbackPacketCalculatorOptions>();
switch (options.type()) {
case CallbackPacketCalculatorOptions::VECTOR_PACKET:
@@ -60,17 +60,17 @@ class CallbackPacketCalculator : public CalculatorBase {
.Set<std::function<void(const Packet&)>>();
break;
default:
return ::mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
return mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
<< "Invalid type of callback to produce.";
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status Open(CalculatorContext* cc) override {
mediapipe::Status Open(CalculatorContext* cc) override {
const auto& options = cc->Options<CallbackPacketCalculatorOptions>();
void* ptr;
if (sscanf(options.pointer().c_str(), "%p", &ptr) != 1) {
return ::mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
return mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
<< "Stored pointer value in options is invalid.";
}
switch (options.type()) {
@@ -87,14 +87,14 @@ class CallbackPacketCalculator : public CalculatorBase {
std::placeholders::_1)));
break;
default:
return ::mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
return mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
<< "Invalid type to dump into.";
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status Process(CalculatorContext* cc) override {
return ::mediapipe::OkStatus();
mediapipe::Status Process(CalculatorContext* cc) override {
return mediapipe::OkStatus();
}
};
mediapipe/calculators/tensor/BUILD
+60 -1
View File
@@ -61,7 +61,7 @@ cc_library(
"@com_google_absl//absl/memory",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/formats:tensor",
"//mediapipe/util:resource_util",
"//mediapipe/util/tflite:tflite_model_loader",
"//mediapipe/util/tflite:config",
"@org_tensorflow//tensorflow/lite:framework",
"@org_tensorflow//tensorflow/lite/delegates/xnnpack:xnnpack_delegate",
@@ -293,6 +293,16 @@ cc_library(
alwayslink = 1,
)
mediapipe_proto_library(
name = "tensors_to_floats_calculator_proto",
srcs = ["tensors_to_floats_calculator.proto"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
cc_library(
name = "tensors_to_floats_calculator",
srcs = ["tensors_to_floats_calculator.cc"],
@@ -305,6 +315,7 @@ cc_library(
}),
visibility = ["//visibility:public"],
deps = [
":tensors_to_floats_calculator_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/formats:tensor",
"//mediapipe/framework/port:ret_check",
@@ -312,6 +323,23 @@ cc_library(
alwayslink = 1,
)
cc_test(
name = "tensors_to_floats_calculator_test",
srcs = ["tensors_to_floats_calculator_test.cc"],
deps = [
":tensors_to_floats_calculator",
":tensors_to_floats_calculator_cc_proto",
"//mediapipe/framework:calculator_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:calculator_runner",
"//mediapipe/framework/formats:tensor",
"//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:parse_text_proto",
"@com_google_absl//absl/memory",
"@com_google_googletest//:gtest_main",
],
)
cc_library(
name = "tensors_to_classification_calculator",
srcs = ["tensors_to_classification_calculator.cc"],
@@ -445,15 +473,22 @@ cc_test(
data = [
"testdata/image_to_tensor/input.jpg",
"testdata/image_to_tensor/large_sub_rect.png",
"testdata/image_to_tensor/large_sub_rect_border_zero.png",
"testdata/image_to_tensor/large_sub_rect_keep_aspect.png",
"testdata/image_to_tensor/large_sub_rect_keep_aspect_border_zero.png",
"testdata/image_to_tensor/large_sub_rect_keep_aspect_with_rotation.png",
"testdata/image_to_tensor/large_sub_rect_keep_aspect_with_rotation_border_zero.png",
"testdata/image_to_tensor/medium_sub_rect_keep_aspect.png",
"testdata/image_to_tensor/medium_sub_rect_keep_aspect_border_zero.png",
"testdata/image_to_tensor/medium_sub_rect_keep_aspect_with_rotation.png",
"testdata/image_to_tensor/medium_sub_rect_keep_aspect_with_rotation_border_zero.png",
"testdata/image_to_tensor/medium_sub_rect_with_rotation.png",
"testdata/image_to_tensor/medium_sub_rect_with_rotation_border_zero.png",
"testdata/image_to_tensor/noop_except_range.png",
],
deps = [
":image_to_tensor_calculator",
":image_to_tensor_converter",
":image_to_tensor_utils",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:calculator_runner",
@@ -479,6 +514,13 @@ cc_test(
cc_library(
name = "image_to_tensor_converter",
hdrs = ["image_to_tensor_converter.h"],
copts = select({
"//mediapipe:apple": [
"-x objective-c++",
"-fobjc-arc", # enable reference-counting
],
"//conditions:default": [],
}),
deps = [
":image_to_tensor_utils",
"//mediapipe/framework:packet",
@@ -521,6 +563,7 @@ cc_library(
"//mediapipe:apple": [],
"//conditions:default": [
":image_to_tensor_converter",
":image_to_tensor_converter_gl_utils",
":image_to_tensor_utils",
"@com_google_absl//absl/strings",
"//mediapipe/framework:calculator_framework",
@@ -553,6 +596,7 @@ cc_library(
"//mediapipe/gpu:disable_gpu": [],
"//conditions:default": [
":image_to_tensor_converter",
":image_to_tensor_converter_gl_utils",
":image_to_tensor_utils",
"@com_google_absl//absl/strings",
"//mediapipe/framework:calculator_framework",
@@ -568,6 +612,21 @@ cc_library(
}),
)
cc_library(
name = "image_to_tensor_converter_gl_utils",
srcs = ["image_to_tensor_converter_gl_utils.cc"],
hdrs = ["image_to_tensor_converter_gl_utils.h"],
deps = ["//mediapipe/framework:port"] + select({
"//mediapipe/gpu:disable_gpu": [],
"//conditions:default": [
"//mediapipe/gpu:gl_base",
"//mediapipe/gpu:gl_context",
"//mediapipe/framework/port:status",
"//mediapipe/framework/port:statusor",
],
}),
)
cc_library(
name = "image_to_tensor_converter_metal",
srcs = ["image_to_tensor_converter_metal.cc"],
mediapipe/calculators/tensor/image_to_tensor_calculator.cc
+31 -15
View File
@@ -14,6 +14,7 @@
#include <array>
#include <memory>
#include <vector>
#include "mediapipe/calculators/tensor/image_to_tensor_calculator.pb.h"
#include "mediapipe/calculators/tensor/image_to_tensor_converter.h"
@@ -111,7 +112,7 @@ namespace mediapipe {
// }
class ImageToTensorCalculator : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
static mediapipe::Status GetContract(CalculatorContract* cc) {
const auto& options =
cc->Options<mediapipe::ImageToTensorCalculatorOptions>();
@@ -157,10 +158,10 @@ class ImageToTensorCalculator : public CalculatorBase {
#endif // MEDIAPIPE_DISABLE_GPU
}
cc->Outputs().Tag(kOutput).Set<std::vector<Tensor>>();
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status Open(CalculatorContext* cc) {
mediapipe::Status Open(CalculatorContext* cc) {
// Makes sure outputs' next timestamp bound update is handled automatically
// by the framework.
cc->SetOffset(TimestampDiff(0));
@@ -171,40 +172,42 @@ class ImageToTensorCalculator : public CalculatorBase {
range_max_ = options_.output_tensor_float_range().max();
if (cc->Inputs().HasTag(kInputCpu)) {
ASSIGN_OR_RETURN(converter_, CreateOpenCvConverter(cc));
ASSIGN_OR_RETURN(converter_, CreateOpenCvConverter(cc, GetBorderMode()));
} else {
#if MEDIAPIPE_DISABLE_GPU
return mediapipe::UnimplementedError("GPU processing is disabled");
#else
#if MEDIAPIPE_METAL_ENABLED
ASSIGN_OR_RETURN(converter_, CreateMetalConverter(cc));
ASSIGN_OR_RETURN(converter_, CreateMetalConverter(cc, GetBorderMode()));
#elif MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31
ASSIGN_OR_RETURN(converter_, CreateImageToGlBufferTensorConverter(
cc, DoesInputStartAtBottom()));
ASSIGN_OR_RETURN(converter_,
CreateImageToGlBufferTensorConverter(
cc, DoesInputStartAtBottom(), GetBorderMode()));
#else
ASSIGN_OR_RETURN(converter_, CreateImageToGlTextureTensorConverter(
cc, DoesInputStartAtBottom()));
ASSIGN_OR_RETURN(converter_,
CreateImageToGlTextureTensorConverter(
cc, DoesInputStartAtBottom(), GetBorderMode()));
#endif // MEDIAPIPE_METAL_ENABLED
#endif // MEDIAPIPE_DISABLE_GPU
}
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
::mediapipe::Status Process(CalculatorContext* cc) {
mediapipe::Status Process(CalculatorContext* cc) {
const InputStreamShard& input = cc->Inputs().Tag(
cc->Inputs().HasTag(kInputCpu) ? kInputCpu : kInputGpu);
if (input.IsEmpty()) {
// Timestamp bound update happens automatically. (See Open().)
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
absl::optional<mediapipe::NormalizedRect> norm_rect;
if (cc->Inputs().HasTag(kInputNormRect)) {
if (cc->Inputs().Tag(kInputNormRect).IsEmpty()) {
// Timestamp bound update happens automatically. (See Open().)
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
norm_rect =
cc->Inputs().Tag(kInputNormRect).Get<mediapipe::NormalizedRect>();
@@ -216,7 +219,7 @@ class ImageToTensorCalculator : public CalculatorBase {
// NOTE: usage of sentinel rects should be avoided.
DLOG(WARNING)
<< "Updating timestamp bound in response to a sentinel rect";
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
}
@@ -254,7 +257,7 @@ class ImageToTensorCalculator : public CalculatorBase {
MakePacket<std::vector<Tensor>>(std::move(result))
.At(cc->InputTimestamp()));
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
private:
@@ -262,6 +265,19 @@ class ImageToTensorCalculator : public CalculatorBase {
return options_.gpu_origin() != mediapipe::GpuOrigin_Mode_TOP_LEFT;
}
BorderMode GetBorderMode() {
switch (options_.border_mode()) {
case mediapipe::
ImageToTensorCalculatorOptions_BorderMode_BORDER_UNSPECIFIED:
return BorderMode::kReplicate;
case mediapipe::ImageToTensorCalculatorOptions_BorderMode_BORDER_ZERO:
return BorderMode::kZero;
case mediapipe::
ImageToTensorCalculatorOptions_BorderMode_BORDER_REPLICATE:
return BorderMode::kReplicate;
}
}
std::unique_ptr<ImageToTensorConverter> converter_;
mediapipe::ImageToTensorCalculatorOptions options_;
int output_width_ = 0;
mediapipe/calculators/tensor/image_to_tensor_calculator.proto
+15
View File
@@ -44,6 +44,13 @@ message ImageToTensorCalculatorOptions {
optional float max = 2;
}
// Pixel extrapolation methods. See @border_mode.
enum BorderMode {
BORDER_UNSPECIFIED = 0;
BORDER_ZERO = 1;
BORDER_REPLICATE = 2;
}
optional int32 output_tensor_width = 1;
optional int32 output_tensor_height = 2;
@@ -61,4 +68,12 @@ message ImageToTensorCalculatorOptions {
// to be flipped vertically as tensors are expected to start at top.
// (DEFAULT or unset interpreted as CONVENTIONAL.)
optional GpuOrigin.Mode gpu_origin = 5;
// Pixel extrapolation method.
// When converting image to tensor it may happen that tensor needs to read
// pixels outside image boundaries. Border mode helps to specify how such
// pixels will be calculated.
//
// BORDER_REPLICATE is used by default.
optional BorderMode border_mode = 6;
}
mediapipe/calculators/tensor/image_to_tensor_calculator_test.cc
+157 -13
View File
@@ -17,6 +17,7 @@
#include "absl/memory/memory.h"
#include "absl/strings/substitute.h"
#include "mediapipe/calculators/tensor/image_to_tensor_converter.h"
#include "mediapipe/calculators/tensor/image_to_tensor_utils.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_runner.h"
@@ -55,7 +56,19 @@ cv::Mat GetRgba(absl::string_view path) {
// No processing/assertions should be done after the function is invoked.
void RunTest(cv::Mat input, cv::Mat expected_result, float range_min,
float range_max, int tensor_width, int tensor_height,
bool keep_aspect, const mediapipe::NormalizedRect& roi) {
bool keep_aspect, absl::optional<BorderMode> border_mode,
const mediapipe::NormalizedRect& roi) {
std::string border_mode_str;
if (border_mode) {
switch (*border_mode) {
case BorderMode::kReplicate:
border_mode_str = "border_mode: BORDER_REPLICATE";
break;
case BorderMode::kZero:
border_mode_str = "border_mode: BORDER_ZERO";
break;
}
}
auto graph_config = mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
absl::Substitute(R"(
input_stream: "input_image"
@@ -67,13 +80,14 @@ void RunTest(cv::Mat input, cv::Mat expected_result, float range_min,
output_stream: "TENSORS:tensor"
options {
[mediapipe.ImageToTensorCalculatorOptions.ext] {
output_tensor_width: $0
output_tensor_height: $1
keep_aspect_ratio: $4
output_tensor_float_range {
output_tensor_width: $0
output_tensor_height: $1
keep_aspect_ratio: $4
output_tensor_float_range {
min: $2
max: $3
}
$5 # border mode
}
}
}
@@ -82,7 +96,8 @@ void RunTest(cv::Mat input, cv::Mat expected_result, float range_min,
/*$1=*/tensor_height,
/*$2=*/range_min,
/*$3=*/range_max,
/*$4=*/keep_aspect ? "true" : "false"));
/*$4=*/keep_aspect ? "true" : "false",
/*$5=*/border_mode_str));
std::vector<Packet> output_packets;
tool::AddVectorSink("tensor", &graph_config, &output_packets);
@@ -151,7 +166,26 @@ TEST(ImageToTensorCalculatorTest, MediumSubRectKeepAspect) {
"tensor/testdata/image_to_tensor/medium_sub_rect_keep_aspect.png"),
/*range_min=*/0.0f,
/*range_max=*/1.0f,
/*tensor_width=*/256, /*tensor_height=*/256, /*keep_aspect=*/true, roi);
/*tensor_width=*/256, /*tensor_height=*/256, /*keep_aspect=*/true,
/*border mode*/ {}, roi);
}
TEST(ImageToTensorCalculatorTest, MediumSubRectKeepAspectBorderZero) {
mediapipe::NormalizedRect roi;
roi.set_x_center(0.65f);
roi.set_y_center(0.4f);
roi.set_width(0.5f);
roi.set_height(0.5f);
roi.set_rotation(0);
RunTest(GetRgb("/mediapipe/calculators/"
"tensor/testdata/image_to_tensor/input.jpg"),
GetRgb("/mediapipe/calculators/"
"tensor/testdata/image_to_tensor/"
"medium_sub_rect_keep_aspect_border_zero.png"),
/*range_min=*/0.0f,
/*range_max=*/1.0f,
/*tensor_width=*/256, /*tensor_height=*/256, /*keep_aspect=*/true,
BorderMode::kZero, roi);
}
TEST(ImageToTensorCalculatorTest, MediumSubRectKeepAspectWithRotation) {
@@ -168,7 +202,25 @@ TEST(ImageToTensorCalculatorTest, MediumSubRectKeepAspectWithRotation) {
"medium_sub_rect_keep_aspect_with_rotation.png"),
/*range_min=*/0.0f, /*range_max=*/1.0f,
/*tensor_width=*/256, /*tensor_height=*/256, /*keep_aspect=*/true,
roi);
BorderMode::kReplicate, roi);
}
TEST(ImageToTensorCalculatorTest,
MediumSubRectKeepAspectWithRotationBorderZero) {
mediapipe::NormalizedRect roi;
roi.set_x_center(0.65f);
roi.set_y_center(0.4f);
roi.set_width(0.5f);
roi.set_height(0.5f);
roi.set_rotation(M_PI * 90.0f / 180.0f);
RunTest(GetRgb("/mediapipe/calculators/"
"tensor/testdata/image_to_tensor/input.jpg"),
GetRgb("/mediapipe/calculators/"
"tensor/testdata/image_to_tensor/"
"medium_sub_rect_keep_aspect_with_rotation_border_zero.png"),
/*range_min=*/0.0f, /*range_max=*/1.0f,
/*tensor_width=*/256, /*tensor_height=*/256, /*keep_aspect=*/true,
BorderMode::kZero, roi);
}
TEST(ImageToTensorCalculatorTest, MediumSubRectWithRotation) {
@@ -186,7 +238,26 @@ TEST(ImageToTensorCalculatorTest, MediumSubRectWithRotation) {
"tensor/testdata/image_to_tensor/medium_sub_rect_with_rotation.png"),
/*range_min=*/-1.0f,
/*range_max=*/1.0f,
/*tensor_width=*/256, /*tensor_height=*/256, /*keep_aspect=*/false, roi);
/*tensor_width=*/256, /*tensor_height=*/256, /*keep_aspect=*/false,
BorderMode::kReplicate, roi);
}
TEST(ImageToTensorCalculatorTest, MediumSubRectWithRotationBorderZero) {
mediapipe::NormalizedRect roi;
roi.set_x_center(0.65f);
roi.set_y_center(0.4f);
roi.set_width(0.5f);
roi.set_height(0.5f);
roi.set_rotation(M_PI * -45.0f / 180.0f);
RunTest(GetRgb("/mediapipe/calculators/"
"tensor/testdata/image_to_tensor/input.jpg"),
GetRgb("/mediapipe/calculators/"
"tensor/testdata/image_to_tensor/"
"medium_sub_rect_with_rotation_border_zero.png"),
/*range_min=*/-1.0f,
/*range_max=*/1.0f,
/*tensor_width=*/256, /*tensor_height=*/256, /*keep_aspect=*/false,
BorderMode::kZero, roi);
}
TEST(ImageToTensorCalculatorTest, LargeSubRect) {
@@ -203,7 +274,25 @@ TEST(ImageToTensorCalculatorTest, LargeSubRect) {
/*range_min=*/0.0f,
/*range_max=*/1.0f,
/*tensor_width=*/128, /*tensor_height=*/128, /*keep_aspect=*/false,
roi);
BorderMode::kReplicate, roi);
}
TEST(ImageToTensorCalculatorTest, LargeSubRectBorderZero) {
mediapipe::NormalizedRect roi;
roi.set_x_center(0.5f);
roi.set_y_center(0.5f);
roi.set_width(1.5f);
roi.set_height(1.1f);
roi.set_rotation(0);
RunTest(
GetRgb("/mediapipe/calculators/"
"tensor/testdata/image_to_tensor/input.jpg"),
GetRgb("/mediapipe/calculators/"
"tensor/testdata/image_to_tensor/large_sub_rect_border_zero.png"),
/*range_min=*/0.0f,
/*range_max=*/1.0f,
/*tensor_width=*/128, /*tensor_height=*/128, /*keep_aspect=*/false,
BorderMode::kZero, roi);
}
TEST(ImageToTensorCalculatorTest, LargeSubRectKeepAspect) {
@@ -220,7 +309,26 @@ TEST(ImageToTensorCalculatorTest, LargeSubRectKeepAspect) {
"tensor/testdata/image_to_tensor/large_sub_rect_keep_aspect.png"),
/*range_min=*/0.0f,
/*range_max=*/1.0f,
/*tensor_width=*/128, /*tensor_height=*/128, /*keep_aspect=*/true, roi);
/*tensor_width=*/128, /*tensor_height=*/128, /*keep_aspect=*/true,
BorderMode::kReplicate, roi);
}
TEST(ImageToTensorCalculatorTest, LargeSubRectKeepAspectBorderZero) {
mediapipe::NormalizedRect roi;
roi.set_x_center(0.5f);
roi.set_y_center(0.5f);
roi.set_width(1.5f);
roi.set_height(1.1f);
roi.set_rotation(0);
RunTest(GetRgb("/mediapipe/calculators/"
"tensor/testdata/image_to_tensor/input.jpg"),
GetRgb("/mediapipe/calculators/"
"tensor/testdata/image_to_tensor/"
"large_sub_rect_keep_aspect_border_zero.png"),
/*range_min=*/0.0f,
/*range_max=*/1.0f,
/*tensor_width=*/128, /*tensor_height=*/128, /*keep_aspect=*/true,
BorderMode::kZero, roi);
}
TEST(ImageToTensorCalculatorTest, LargeSubRectKeepAspectWithRotation) {
@@ -238,7 +346,26 @@ TEST(ImageToTensorCalculatorTest, LargeSubRectKeepAspectWithRotation) {
/*range_min=*/0.0f,
/*range_max=*/1.0f,
/*tensor_width=*/128, /*tensor_height=*/128, /*keep_aspect=*/true,
roi);
/*border_mode=*/{}, roi);
}
TEST(ImageToTensorCalculatorTest,
LargeSubRectKeepAspectWithRotationBorderZero) {
mediapipe::NormalizedRect roi;
roi.set_x_center(0.5f);
roi.set_y_center(0.5f);
roi.set_width(1.5f);
roi.set_height(1.1f);
roi.set_rotation(M_PI * -15.0f / 180.0f);
RunTest(GetRgba("/mediapipe/calculators/"
"tensor/testdata/image_to_tensor/input.jpg"),
GetRgb("/mediapipe/calculators/"
"tensor/testdata/image_to_tensor/"
"large_sub_rect_keep_aspect_with_rotation_border_zero.png"),
/*range_min=*/0.0f,
/*range_max=*/1.0f,
/*tensor_width=*/128, /*tensor_height=*/128, /*keep_aspect=*/true,
/*border_mode=*/BorderMode::kZero, roi);
}
TEST(ImageToTensorCalculatorTest, NoOpExceptRange) {
@@ -255,7 +382,24 @@ TEST(ImageToTensorCalculatorTest, NoOpExceptRange) {
/*range_min=*/0.0f,
/*range_max=*/1.0f,
/*tensor_width=*/64, /*tensor_height=*/128, /*keep_aspect=*/true,
roi);
BorderMode::kReplicate, roi);
}
TEST(ImageToTensorCalculatorTest, NoOpExceptRangeBorderZero) {
mediapipe::NormalizedRect roi;
roi.set_x_center(0.5f);
roi.set_y_center(0.5f);
roi.set_width(1.0f);
roi.set_height(1.0f);
roi.set_rotation(0);
RunTest(GetRgba("/mediapipe/calculators/"
"tensor/testdata/image_to_tensor/input.jpg"),
GetRgb("/mediapipe/calculators/"
"tensor/testdata/image_to_tensor/noop_except_range.png"),
/*range_min=*/0.0f,
/*range_max=*/1.0f,
/*tensor_width=*/64, /*tensor_height=*/128, /*keep_aspect=*/true,
BorderMode::kZero, roi);
}
} // namespace
mediapipe/calculators/tensor/image_to_tensor_converter.h
+11 -5
View File
@@ -27,6 +27,12 @@ struct Size {
int height;
};
// Pixel extrapolation method.
// When converting image to tensor it may happen that tensor needs to read
// pixels outside image boundaries. Border mode helps to specify how such pixels
// will be calculated.
enum class BorderMode { kZero, kReplicate };
// Converts image to tensor.
class ImageToTensorConverter {
public:
@@ -41,11 +47,11 @@ class ImageToTensorConverter {
// @output_dims dimensions of output tensor.
// @range_min/max describes output tensor range image pixels should converted
// to.
virtual ::mediapipe::StatusOr<Tensor> Convert(const Packet& image_packet,
const RotatedRect& roi,
const Size& output_dims,
float range_min,
float range_max) = 0;
virtual mediapipe::StatusOr<Tensor> Convert(const Packet& image_packet,
const RotatedRect& roi,
const Size& output_dims,
float range_min,
float range_max) = 0;
};
} // namespace mediapipe
mediapipe/calculators/tensor/image_to_tensor_converter_gl_buffer.cc
+96 -82
View File
@@ -14,6 +14,8 @@
#include "mediapipe/calculators/tensor/image_to_tensor_converter_gl_buffer.h"
#include "mediapipe/framework/port.h"
#if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31
#include <array>
@@ -22,6 +24,7 @@
#include "absl/strings/str_cat.h"
#include "mediapipe/calculators/tensor/image_to_tensor_converter.h"
#include "mediapipe/calculators/tensor/image_to_tensor_converter_gl_utils.h"
#include "mediapipe/calculators/tensor/image_to_tensor_utils.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/tensor.h"
@@ -51,7 +54,7 @@ class SubRectExtractorGl {
public:
// Extracts a region defined by @sub_rect, removes A channel, transforms input
// pixels as alpha * x + beta and resizes result into destination.
::mediapipe::Status ExtractSubRectToBuffer(
mediapipe::Status ExtractSubRectToBuffer(
const tflite::gpu::gl::GlTexture& texture,
const tflite::gpu::HW& texture_size, const RotatedRect& sub_rect,
bool flip_horizontaly, float alpha, float beta,
@@ -59,20 +62,28 @@ class SubRectExtractorGl {
tflite::gpu::gl::CommandQueue* command_queue,
tflite::gpu::gl::GlBuffer* destination);
static ::mediapipe::StatusOr<SubRectExtractorGl> Create(
bool input_starts_at_bottom);
static mediapipe::StatusOr<SubRectExtractorGl> Create(
const mediapipe::GlContext& gl_context, bool input_starts_at_bottom,
BorderMode border_mode);
private:
explicit SubRectExtractorGl(tflite::gpu::gl::GlProgram program,
tflite::gpu::uint3 workgroup_size)
: program_(std::move(program)), workgroup_size_(workgroup_size) {}
tflite::gpu::uint3 workgroup_size,
bool use_custom_zero_border,
BorderMode border_mode)
: program_(std::move(program)),
workgroup_size_(workgroup_size),
use_custom_zero_border_(use_custom_zero_border),
border_mode_(border_mode) {}
tflite::gpu::gl::GlProgram program_;
tflite::gpu::uint3 workgroup_size_;
bool use_custom_zero_border_ = false;
BorderMode border_mode_ = BorderMode::kReplicate;
};
::mediapipe::Status SetMat4x4(const tflite::gpu::gl::GlProgram& program,
const std::string& name, float* data) {
mediapipe::Status SetMat4x4(const tflite::gpu::gl::GlProgram& program,
const std::string& name, float* data) {
GLint uniform_id;
MP_RETURN_IF_ERROR(TFLITE_GPU_CALL_GL(glGetUniformLocation, &uniform_id,
program.id(), name.c_str()));
@@ -80,44 +91,6 @@ class SubRectExtractorGl {
1, GL_TRUE, data);
}
class GlParametersOverride {
public:
static ::mediapipe::StatusOr<GlParametersOverride> Create(
const std::vector<std::pair<GLenum, GLint>>& overrides) {
std::vector<GLint> old_values(overrides.size());
for (int i = 0; i < overrides.size(); ++i) {
MP_RETURN_IF_ERROR(TFLITE_GPU_CALL_GL(glGetTexParameteriv, GL_TEXTURE_2D,
overrides[i].first,
&old_values[i]));
if (overrides[i].second != old_values[i]) {
MP_RETURN_IF_ERROR(TFLITE_GPU_CALL_GL(glTexParameteri, GL_TEXTURE_2D,
overrides[i].first,
overrides[i].second));
}
}
return GlParametersOverride(overrides, std::move(old_values));
}
::mediapipe::Status Revert() {
for (int i = 0; i < overrides_.size(); ++i) {
if (overrides_[i].second != old_values_[i]) {
MP_RETURN_IF_ERROR(TFLITE_GPU_CALL_GL(glTexParameteri, GL_TEXTURE_2D,
overrides_[i].first,
old_values_[i]));
}
}
return ::mediapipe::OkStatus();
}
private:
GlParametersOverride(const std::vector<std::pair<GLenum, GLint>>& overrides,
std::vector<GLint> old_values)
: overrides_(overrides), old_values_(std::move(old_values)) {}
std::vector<std::pair<GLenum, GLint>> overrides_;
std::vector<GLint> old_values_;
};
constexpr char kShaderCode[] = R"(
layout(std430) buffer;
@@ -162,6 +135,12 @@ void main() {
#endif // INPUT_STARTS_AT_BOTTOM
vec4 src_value = alpha * texture(input_data, tc.xy) + beta;
#ifdef CUSTOM_ZERO_BORDER_MODE
float out_of_bounds =
float(tc.x < 0.0 || tc.x > 1.0 || tc.y < 0.0 || tc.y > 1.0);
src_value = mix(src_value, vec4(0.0, 0.0, 0.0, 0.0), out_of_bounds);
#endif
int linear_index = gid.y * out_width + gid.x;
// output_data.elements is populated as though it contains vec3 elements.
@@ -172,7 +151,7 @@ void main() {
}
)";
::mediapipe::Status SubRectExtractorGl::ExtractSubRectToBuffer(
mediapipe::Status SubRectExtractorGl::ExtractSubRectToBuffer(
const tflite::gpu::gl::GlTexture& texture,
const tflite::gpu::HW& texture_size, const RotatedRect& texture_sub_rect,
bool flip_horizontaly, float alpha, float beta,
@@ -185,11 +164,27 @@ void main() {
&transform_mat);
MP_RETURN_IF_ERROR(texture.BindAsSampler2D(0));
ASSIGN_OR_RETURN(auto overrides, GlParametersOverride::Create(
{{GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE},
{GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE},
{GL_TEXTURE_MIN_FILTER, GL_LINEAR},
{GL_TEXTURE_MAG_FILTER, GL_LINEAR}}));
// a) Filtering.
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
// b) Clamping.
switch (border_mode_) {
case BorderMode::kReplicate: {
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
break;
}
case BorderMode::kZero: {
if (!use_custom_zero_border_) {
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_BORDER);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_BORDER);
glTexParameterfv(GL_TEXTURE_2D, GL_TEXTURE_BORDER_COLOR,
std::array<float, 4>{0.0f, 0.0f, 0.0f, 0.0f}.data());
}
break;
}
}
MP_RETURN_IF_ERROR(destination->BindToIndex(0));
MP_RETURN_IF_ERROR(program_.SetParameter({"input_data", 0}));
@@ -204,11 +199,21 @@ void main() {
workgroup_size_);
MP_RETURN_IF_ERROR(command_queue->Dispatch(program_, num_workgroups));
return overrides.Revert();
// Resetting to MediaPipe texture param defaults.
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
return mediapipe::OkStatus();
}
::mediapipe::StatusOr<SubRectExtractorGl> SubRectExtractorGl::Create(
bool input_starts_at_bottom) {
mediapipe::StatusOr<SubRectExtractorGl> SubRectExtractorGl::Create(
const mediapipe::GlContext& gl_context, bool input_starts_at_bottom,
BorderMode border_mode) {
bool use_custom_zero_border = border_mode == BorderMode::kZero &&
!IsGlClampToBorderSupported(gl_context);
const tflite::gpu::uint3 workgroup_size = {8, 8, 1};
std::string starts_at_bottom_def;
if (input_starts_at_bottom) {
@@ -216,9 +221,15 @@ void main() {
#define INPUT_STARTS_AT_BOTTOM;
)";
}
const std::string full_shader_source =
absl::StrCat(tflite::gpu::gl::GetShaderHeader(workgroup_size),
starts_at_bottom_def, kShaderCode);
std::string custom_zero_border_mode_def;
if (use_custom_zero_border) {
custom_zero_border_mode_def = R"(
#define CUSTOM_ZERO_BORDER_MODE
)";
}
const std::string full_shader_source = absl::StrCat(
tflite::gpu::gl::GetShaderHeader(workgroup_size), starts_at_bottom_def,
custom_zero_border_mode_def, kShaderCode);
tflite::gpu::gl::GlShader shader;
MP_RETURN_IF_ERROR(tflite::gpu::gl::GlShader::CompileShader(
@@ -227,27 +238,30 @@ void main() {
MP_RETURN_IF_ERROR(
tflite::gpu::gl::GlProgram::CreateWithShader(shader, &program));
return SubRectExtractorGl(std::move(program), workgroup_size);
return SubRectExtractorGl(std::move(program), workgroup_size,
use_custom_zero_border, border_mode);
}
class GlProcessor : public ImageToTensorConverter {
public:
::mediapipe::Status Init(CalculatorContext* cc, bool input_starts_at_bottom) {
mediapipe::Status Init(CalculatorContext* cc, bool input_starts_at_bottom,
BorderMode border_mode) {
MP_RETURN_IF_ERROR(gl_helper_.Open(cc));
return gl_helper_.RunInGlContext(
[this, input_starts_at_bottom]() -> ::mediapipe::Status {
tflite::gpu::GpuInfo gpu_info;
MP_RETURN_IF_ERROR(tflite::gpu::gl::RequestGpuInfo(&gpu_info));
RET_CHECK(tflite::gpu::IsOpenGl31OrAbove(gpu_info))
<< "OpenGL ES 3.1 is required.";
command_queue_ = tflite::gpu::gl::NewCommandQueue(gpu_info);
return gl_helper_.RunInGlContext([this, input_starts_at_bottom,
border_mode]() -> mediapipe::Status {
tflite::gpu::GpuInfo gpu_info;
MP_RETURN_IF_ERROR(tflite::gpu::gl::RequestGpuInfo(&gpu_info));
RET_CHECK(gpu_info.IsApiOpenGl31OrAbove())
<< "OpenGL ES 3.1 is required.";
command_queue_ = tflite::gpu::gl::NewCommandQueue(gpu_info);
ASSIGN_OR_RETURN(auto extractor,
SubRectExtractorGl::Create(input_starts_at_bottom));
extractor_ =
absl::make_unique<SubRectExtractorGl>(std::move(extractor));
return ::mediapipe::OkStatus();
});
ASSIGN_OR_RETURN(
auto extractor,
SubRectExtractorGl::Create(gl_helper_.GetGlContext(),
input_starts_at_bottom, border_mode));
extractor_ = absl::make_unique<SubRectExtractorGl>(std::move(extractor));
return mediapipe::OkStatus();
});
}
Size GetImageSize(const Packet& image_packet) override {
@@ -255,11 +269,10 @@ class GlProcessor : public ImageToTensorConverter {
return {image.width(), image.height()};
}
::mediapipe::StatusOr<Tensor> Convert(const Packet& image_packet,
const RotatedRect& roi,
const Size& output_dims,
float range_min,
float range_max) override {
mediapipe::StatusOr<Tensor> Convert(const Packet& image_packet,
const RotatedRect& roi,
const Size& output_dims, float range_min,
float range_max) override {
const auto& input = image_packet.Get<mediapipe::GpuBuffer>();
if (input.format() != mediapipe::GpuBufferFormat::kBGRA32) {
return InvalidArgumentError(
@@ -273,7 +286,7 @@ class GlProcessor : public ImageToTensorConverter {
MP_RETURN_IF_ERROR(gl_helper_.RunInGlContext(
[this, &tensor, &input, &roi, &output_dims, range_min,
range_max]() -> ::mediapipe::Status {
range_max]() -> mediapipe::Status {
constexpr int kRgbaNumChannels = 4;
auto source_texture = gl_helper_.CreateSourceTexture(input);
tflite::gpu::gl::GlTexture input_texture(
@@ -303,7 +316,7 @@ class GlProcessor : public ImageToTensorConverter {
tflite::gpu::HW(output_dims.height, output_dims.width),
command_queue_.get(), &output));
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}));
return tensor;
@@ -325,11 +338,12 @@ class GlProcessor : public ImageToTensorConverter {
} // namespace
::mediapipe::StatusOr<std::unique_ptr<ImageToTensorConverter>>
mediapipe::StatusOr<std::unique_ptr<ImageToTensorConverter>>
CreateImageToGlBufferTensorConverter(CalculatorContext* cc,
bool input_starts_at_bottom) {
bool input_starts_at_bottom,
BorderMode border_mode) {
auto result = absl::make_unique<GlProcessor>();
MP_RETURN_IF_ERROR(result->Init(cc, input_starts_at_bottom));
MP_RETURN_IF_ERROR(result->Init(cc, input_starts_at_bottom, border_mode));
// Simply "return std::move(result)" failed to build on macOS with bazel.
return std::unique_ptr<ImageToTensorConverter>(std::move(result));
mediapipe/calculators/tensor/image_to_tensor_converter_gl_buffer.h
+3 -2
View File
@@ -30,9 +30,10 @@ namespace mediapipe {
// Creates image to tensor (represented as OpenGL buffer) converter.
// NOTE: mediapipe::GlCalculatorHelper::UpdateContract invocation must precede
// converter creation.
::mediapipe::StatusOr<std::unique_ptr<ImageToTensorConverter>>
mediapipe::StatusOr<std::unique_ptr<ImageToTensorConverter>>
CreateImageToGlBufferTensorConverter(CalculatorContext* cc,
bool input_starts_at_bottom);
bool input_starts_at_bottom,
BorderMode border_mode);
} // namespace mediapipe
mediapipe/calculators/tensor/image_to_tensor_converter_gl_texture.cc
+76 -63
View File
@@ -24,6 +24,7 @@
#include "absl/strings/str_cat.h"
#include "mediapipe/calculators/tensor/image_to_tensor_converter.h"
#include "mediapipe/calculators/tensor/image_to_tensor_converter_gl_utils.h"
#include "mediapipe/calculators/tensor/image_to_tensor_utils.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/tensor.h"
@@ -40,48 +41,22 @@ namespace mediapipe {
namespace {
class GlParametersOverride {
public:
static ::mediapipe::StatusOr<GlParametersOverride> Create(
const std::vector<std::pair<GLenum, GLint>>& overrides) {
std::vector<GLint> old_values(overrides.size());
for (int i = 0; i < overrides.size(); ++i) {
glGetTexParameteriv(GL_TEXTURE_2D, overrides[i].first, &old_values[i]);
if (overrides[i].second != old_values[i]) {
glTexParameteri(GL_TEXTURE_2D, overrides[i].first, overrides[i].second);
}
}
return GlParametersOverride(overrides, std::move(old_values));
}
::mediapipe::Status Revert() {
for (int i = 0; i < overrides_.size(); ++i) {
if (overrides_[i].second != old_values_[i]) {
glTexParameteri(GL_TEXTURE_2D, overrides_[i].first, old_values_[i]);
}
}
return ::mediapipe::OkStatus();
}
private:
GlParametersOverride(const std::vector<std::pair<GLenum, GLint>>& overrides,
std::vector<GLint> old_values)
: overrides_(overrides), old_values_(std::move(old_values)) {}
std::vector<std::pair<GLenum, GLint>> overrides_;
std::vector<GLint> old_values_;
};
constexpr int kAttribVertex = 0;
constexpr int kAttribTexturePosition = 1;
constexpr int kNumAttributes = 2;
class GlProcessor : public ImageToTensorConverter {
public:
::mediapipe::Status Init(CalculatorContext* cc, bool input_starts_at_bottom) {
mediapipe::Status Init(CalculatorContext* cc, bool input_starts_at_bottom,
BorderMode border_mode) {
MP_RETURN_IF_ERROR(gl_helper_.Open(cc));
return gl_helper_.RunInGlContext([this, input_starts_at_bottom]()
-> ::mediapipe::Status {
return gl_helper_.RunInGlContext([this, input_starts_at_bottom,
border_mode]() -> mediapipe::Status {
use_custom_zero_border_ =
border_mode == BorderMode::kZero &&
!IsGlClampToBorderSupported(gl_helper_.GetGlContext());
border_mode_ = border_mode;
const GLint attr_location[kNumAttributes] = {
kAttribVertex,
kAttribTexturePosition,
@@ -127,23 +102,38 @@ class GlProcessor : public ImageToTensorConverter {
#endif // defined(GL_ES);
void main() {
fragColor = alpha * texture2D(input_texture, sample_coordinate) + beta;
vec4 color = texture2D(input_texture, sample_coordinate);
#ifdef CUSTOM_ZERO_BORDER_MODE
float out_of_bounds =
float(sample_coordinate.x < 0.0 || sample_coordinate.x > 1.0 ||
sample_coordinate.y < 0.0 || sample_coordinate.y > 1.0);
color = mix(color, vec4(0.0, 0.0, 0.0, 0.0), out_of_bounds);
#endif // defined(CUSTOM_ZERO_BORDER_MODE)
fragColor = alpha * color + beta;
}
)";
std::string starts_at_bottom_def;
if (input_starts_at_bottom) {
starts_at_bottom_def = R"(
#define INPUT_STARTS_AT_BOTTOM
)";
#define INPUT_STARTS_AT_BOTTOM
)";
}
// Create program and set parameters.
const std::string extract_sub_rect_vertex_src =
absl::StrCat(mediapipe::kMediaPipeVertexShaderPreamble,
starts_at_bottom_def, kExtractSubRectVertexShader);
const std::string extract_sub_rect_frag_src = absl::StrCat(
mediapipe::kMediaPipeFragmentShaderPreamble, kExtractSubRectFragBody);
std::string custom_zero_border_mode_def;
if (use_custom_zero_border_) {
custom_zero_border_mode_def = R"(
#define CUSTOM_ZERO_BORDER_MODE
)";
}
const std::string extract_sub_rect_frag_src =
absl::StrCat(mediapipe::kMediaPipeFragmentShaderPreamble,
custom_zero_border_mode_def, kExtractSubRectFragBody);
mediapipe::GlhCreateProgram(extract_sub_rect_vertex_src.c_str(),
extract_sub_rect_frag_src.c_str(),
kNumAttributes, &attr_name[0], attr_location,
@@ -174,7 +164,7 @@ class GlProcessor : public ImageToTensorConverter {
glBindBuffer(GL_ARRAY_BUFFER, 0);
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
});
}
@@ -183,11 +173,10 @@ class GlProcessor : public ImageToTensorConverter {
return {image.width(), image.height()};
}
::mediapipe::StatusOr<Tensor> Convert(const Packet& image_packet,
const RotatedRect& roi,
const Size& output_dims,
float range_min,
float range_max) override {
mediapipe::StatusOr<Tensor> Convert(const Packet& image_packet,
const RotatedRect& roi,
const Size& output_dims, float range_min,
float range_max) override {
const auto& input = image_packet.Get<mediapipe::GpuBuffer>();
if (input.format() != mediapipe::GpuBufferFormat::kBGRA32) {
return InvalidArgumentError(
@@ -202,7 +191,7 @@ class GlProcessor : public ImageToTensorConverter {
MP_RETURN_IF_ERROR(gl_helper_.RunInGlContext(
[this, &tensor, &input, &roi, &output_dims, range_min,
range_max]() -> ::mediapipe::Status {
range_max]() -> mediapipe::Status {
auto input_texture = gl_helper_.CreateSourceTexture(input);
constexpr float kInputImageRangeMin = 0.0f;
@@ -216,17 +205,17 @@ class GlProcessor : public ImageToTensorConverter {
/*flip_horizontaly=*/false,
transform.scale, transform.offset,
output_dims, &tensor_view));
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}));
return tensor;
}
::mediapipe::Status ExtractSubRect(const mediapipe::GlTexture& texture,
const RotatedRect& sub_rect,
bool flip_horizontaly, float alpha,
float beta, const Size& output_dims,
Tensor::OpenGlTexture2dView* output) {
mediapipe::Status ExtractSubRect(const mediapipe::GlTexture& texture,
const RotatedRect& sub_rect,
bool flip_horizontaly, float alpha,
float beta, const Size& output_dims,
Tensor::OpenGlTexture2dView* output) {
std::array<float, 16> transform_mat;
GetRotatedSubRectToRectTransformMatrix(sub_rect, texture.width(),
texture.height(), flip_horizontaly,
@@ -244,11 +233,27 @@ class GlProcessor : public ImageToTensorConverter {
glActiveTexture(GL_TEXTURE1);
glBindTexture(texture.target(), texture.name());
ASSIGN_OR_RETURN(auto overrides, GlParametersOverride::Create(
{{GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE},
{GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE},
{GL_TEXTURE_MIN_FILTER, GL_LINEAR},
{GL_TEXTURE_MAG_FILTER, GL_LINEAR}}));
// a) Filtering.
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
// b) Clamping.
switch (border_mode_) {
case BorderMode::kReplicate: {
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
break;
}
case BorderMode::kZero: {
if (!use_custom_zero_border_) {
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_BORDER);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_BORDER);
glTexParameterfv(GL_TEXTURE_2D, GL_TEXTURE_BORDER_COLOR,
std::array<float, 4>{0.0f, 0.0f, 0.0f, 0.0f}.data());
}
break;
}
}
glUseProgram(program_);
glUniform1f(alpha_id_, alpha);
@@ -271,7 +276,12 @@ class GlProcessor : public ImageToTensorConverter {
// draw
glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
// cleanup
// Resetting to MediaPipe texture param defaults.
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
glDisableVertexAttribArray(kAttribVertex);
glDisableVertexAttribArray(kAttribTexturePosition);
glBindBuffer(GL_ARRAY_BUFFER, 0);
@@ -282,7 +292,7 @@ class GlProcessor : public ImageToTensorConverter {
glActiveTexture(GL_TEXTURE0);
glBindTexture(GL_TEXTURE_2D, 0);
return overrides.Revert();
return mediapipe::OkStatus();
}
~GlProcessor() override {
@@ -297,6 +307,8 @@ class GlProcessor : public ImageToTensorConverter {
private:
mediapipe::GlCalculatorHelper gl_helper_;
bool use_custom_zero_border_ = false;
BorderMode border_mode_ = BorderMode::kReplicate;
GLuint vao_ = 0;
GLuint vbo_[2] = {0, 0};
GLuint program_ = 0;
@@ -308,11 +320,12 @@ class GlProcessor : public ImageToTensorConverter {
} // namespace
::mediapipe::StatusOr<std::unique_ptr<ImageToTensorConverter>>
mediapipe::StatusOr<std::unique_ptr<ImageToTensorConverter>>
CreateImageToGlTextureTensorConverter(CalculatorContext* cc,
bool input_starts_at_bottom) {
bool input_starts_at_bottom,
BorderMode border_mode) {
auto result = absl::make_unique<GlProcessor>();
MP_RETURN_IF_ERROR(result->Init(cc, input_starts_at_bottom));
MP_RETURN_IF_ERROR(result->Init(cc, input_starts_at_bottom, border_mode));
// Simply "return std::move(result)" failed to build on macOS with bazel.
return std::unique_ptr<ImageToTensorConverter>(std::move(result));
mediapipe/calculators/tensor/image_to_tensor_converter_gl_texture.h
+3 -3
View File
@@ -13,7 +13,6 @@
// limitations under the License.
#ifndef MEDIAPIPE_CALCULATORS_TENSOR_IMAGE_TO_TENSOR_CONVERTER_GL_TEXTURE_H_
#define MEDIAPIPE_CALCULATORS_TENSOR_IMAGE_TO_TENSOR_CONVERTER_GL_TEXTURE_H_
#include "mediapipe/framework/port.h"
@@ -31,9 +30,10 @@ namespace mediapipe {
// Creates image to tensor (represented as OpenGL texture) converter.
// NOTE: mediapipe::GlCalculatorHelper::UpdateContract invocation must precede
// converter creation.
::mediapipe::StatusOr<std::unique_ptr<ImageToTensorConverter>>
mediapipe::StatusOr<std::unique_ptr<ImageToTensorConverter>>
CreateImageToGlTextureTensorConverter(CalculatorContext* cc,
bool input_starts_at_bottom);
bool input_starts_at_bottom,
BorderMode border_mode);
} // namespace mediapipe
mediapipe/calculators/tensor/image_to_tensor_converter_gl_utils.cc
+88
View File
@@ -0,0 +1,88 @@
#include "mediapipe/calculators/tensor/image_to_tensor_converter_gl_utils.h"
#include "mediapipe/framework/port.h"
#if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_20
#include <array>
#include <memory>
#include <vector>
#include "mediapipe/framework/port/status_macros.h"
#include "mediapipe/framework/port/statusor.h"
#include "mediapipe/gpu/gl_base.h"
#include "mediapipe/gpu/gl_context.h"
namespace mediapipe {
namespace {
class GlNoOpOverride : public GlOverride {};
class GlTexParameteriOverride : public GlOverride {
public:
GlTexParameteriOverride(GLenum name, GLint old_value)
: name_(name), old_value_(old_value) {}
~GlTexParameteriOverride() override {
glTexParameteri(GL_TEXTURE_2D, name_, old_value_);
}
private:
GLenum name_;
GLint old_value_;
};
template <int kNumValues>
class GlTexParameterfvOverride : public GlOverride {
public:
GlTexParameterfvOverride(GLenum name,
std::array<float, kNumValues> old_values)
: name_(name), old_values_(std::move(old_values)) {}
~GlTexParameterfvOverride() {
glTexParameterfv(GL_TEXTURE_2D, name_, &old_values_[0]);
}
private:
GLenum name_;
std::array<float, kNumValues> old_values_;
};
} // namespace
std::unique_ptr<GlOverride> OverrideGlTexParametri(GLenum name, GLint value) {
GLint old_value;
glGetTexParameteriv(GL_TEXTURE_2D, name, &old_value);
if (value != old_value) {
glTexParameteri(GL_TEXTURE_2D, name, value);
return {absl::make_unique<GlTexParameteriOverride>(name, old_value)};
}
return {absl::make_unique<GlNoOpOverride>()};
}
template <int kNumValues>
std::unique_ptr<GlOverride> OverrideGlTexParameterfv(
GLenum name, std::array<GLfloat, kNumValues> values) {
std::array<float, kNumValues> old_values;
glGetTexParameterfv(GL_TEXTURE_2D, name, values.data());
if (values != old_values) {
glTexParameterfv(GL_TEXTURE_2D, name, values.data());
return {absl::make_unique<GlTexParameterfvOverride<kNumValues>>(
name, std::move(old_values))};
}
return {absl::make_unique<GlNoOpOverride>()};
}
template std::unique_ptr<GlOverride> OverrideGlTexParameterfv<4>(
GLenum name, std::array<GLfloat, 4> values);
bool IsGlClampToBorderSupported(const mediapipe::GlContext& gl_context) {
return gl_context.gl_major_version() > 3 ||
(gl_context.gl_major_version() == 3 &&
gl_context.gl_minor_version() >= 2);
}
} // namespace mediapipe
#endif // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_20
mediapipe/calculators/tensor/image_to_tensor_converter_gl_utils.h
+45
View File
@@ -0,0 +1,45 @@
#ifndef MEDIAPIPE_CALCULATORS_TENSOR_IMAGE_TO_TENSOR_CONVERTER_GL_UTILS_H_
#define MEDIAPIPE_CALCULATORS_TENSOR_IMAGE_TO_TENSOR_CONVERTER_GL_UTILS_H_
#include "mediapipe/framework/port.h"
#if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_20
#include <array>
#include <memory>
#include <vector>
#include "mediapipe/framework/port/statusor.h"
#include "mediapipe/gpu/gl_base.h"
#include "mediapipe/gpu/gl_context.h"
namespace mediapipe {
// Intended to override and automatically revert various OpenGL attributes.
// (e.g. overriding texture parameters like GL_TEXTURE_MIN_FILTER,
// GL_TEXTURE_MAG_FILTER, etc.)
class GlOverride {
public:
virtual ~GlOverride() = default;
};
// Creates an object that overrides attributes using `glTexParameteri`
// function during construction and reverts them during destruction. See
// `glTexParameteri` for details on @name and @value.
ABSL_MUST_USE_RESULT std::unique_ptr<GlOverride> OverrideGlTexParametri(
GLenum name, GLint value);
// Creates an object that overrides attributes using `glTexParameterfv`
// function during construction and reverts them during destruction. See
// `glTexParameterfv` for details on @name and @values.
template <int kNumValues>
ABSL_MUST_USE_RESULT std::unique_ptr<GlOverride> OverrideGlTexParameterfv(
GLenum name, std::array<GLfloat, kNumValues> values);
bool IsGlClampToBorderSupported(const mediapipe::GlContext& gl_context);
} // namespace mediapipe
#endif // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_20
#endif // MEDIAPIPE_CALCULATORS_TENSOR_IMAGE_TO_TENSOR_CONVERTER_GL_UTILS_H_
mediapipe/calculators/tensor/image_to_tensor_converter_gl_utils_test.cc
+49
View File
@@ -0,0 +1,49 @@
#include "mediapipe/framework/port.h"
#if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_20
#include "mediapipe/calculators/tensor/image_to_tensor_converter_gl_utils.h"
#include "mediapipe/framework/port/gtest.h"
#include "mediapipe/framework/port/status_matchers.h"
#include "mediapipe/gpu/gl_base.h"
#include "mediapipe/gpu/gl_context.h"
namespace mediapipe {
namespace {
TEST(ImageToTensorConverterGlUtilsTest, GlTexParameteriOverrider) {
auto status_or_context = mediapipe::GlContext::Create(nullptr, false);
MP_ASSERT_OK(status_or_context);
auto context = status_or_context.ValueOrDie();
std::vector<GLint> min_filter_changes;
context->Run([&min_filter_changes]() {
GLuint texture = 0;
glGenTextures(1, &texture);
glBindTexture(GL_TEXTURE_2D, texture);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
GLint value = 0;
glGetTexParameteriv(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, &value);
min_filter_changes.push_back(value);
{
auto min_filter_linear =
OverrideGlTexParametri(GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glGetTexParameteriv(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, &value);
min_filter_changes.push_back(value);
// reverter is destroyed automatically reverting previously set value
}
glGetTexParameteriv(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, &value);
min_filter_changes.push_back(value);
});
EXPECT_THAT(min_filter_changes,
testing::ElementsAre(GL_NEAREST, GL_LINEAR, GL_NEAREST));
}
} // namespace
} // namespace mediapipe
#endif // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_20
mediapipe/calculators/tensor/image_to_tensor_converter_metal.cc
+66 -49
View File
@@ -105,8 +105,15 @@ constexpr char kFragmentShader[] = R"(
const float alpha = parameters[0];
const float beta = parameters[1];
#ifdef CLAMP_TO_ZERO
constexpr sampler linear_sampler(address::clamp_to_zero, min_filter::linear,
mag_filter::linear);
#endif // CLAMP_TO_ZERO
#ifdef CLAMP_TO_EDGE
constexpr sampler linear_sampler(address::clamp_to_edge, min_filter::linear,
mag_filter::linear);
#endif // CLAMP_TO_EDGE
Type4 texture_pixel = texture.sample(linear_sampler, vertex_output.uv);
return Type4(alpha * texture_pixel.rgb + beta, 0);
@@ -139,11 +146,12 @@ int GetBytesPerRaw(OutputFormat output_format, const tflite::gpu::HW& size) {
class SubRectExtractorMetal {
public:
static ::mediapipe::StatusOr<std::unique_ptr<SubRectExtractorMetal>> Make(
id<MTLDevice> device, OutputFormat output_format) {
static mediapipe::StatusOr<std::unique_ptr<SubRectExtractorMetal>> Make(
id<MTLDevice> device, OutputFormat output_format,
BorderMode border_mode) {
id<MTLRenderPipelineState> pipeline_state;
MP_RETURN_IF_ERROR(SubRectExtractorMetal::MakePipelineState(
device, output_format, &pipeline_state));
device, output_format, border_mode, &pipeline_state));
return absl::make_unique<SubRectExtractorMetal>(device, pipeline_state,
output_format);
@@ -164,19 +172,14 @@ class SubRectExtractorMetal {
[device_ newBufferWithBytes:kBasicTextureVertices
length:sizeof(kBasicTextureVertices)
options:MTLResourceOptionCPUCacheModeDefault];
transform_mat_buffer_ =
[device_ newBufferWithBytes:&transform_mat_
length:sizeof(transform_mat_)
options:MTLResourceOptionCPUCacheModeDefault];
}
::mediapipe::Status Execute(id<MTLTexture> input_texture,
const RotatedRect& sub_rect,
bool flip_horizontaly, float alpha, float beta,
const tflite::gpu::HW& destination_size,
id<MTLCommandBuffer> command_buffer,
id<MTLBuffer> destination) {
mediapipe::Status Execute(id<MTLTexture> input_texture,
const RotatedRect& sub_rect, bool flip_horizontaly,
float alpha, float beta,
const tflite::gpu::HW& destination_size,
id<MTLCommandBuffer> command_buffer,
id<MTLBuffer> destination) {
auto output_texture = MTLTextureWithBuffer(destination_size, destination);
return InternalExecute(input_texture, sub_rect, flip_horizontaly, alpha,
beta, destination_size, command_buffer,
@@ -202,23 +205,26 @@ class SubRectExtractorMetal {
return texture;
}
::mediapipe::Status InternalExecute(id<MTLTexture> input_texture,
const RotatedRect& sub_rect,
bool flip_horizontaly, float alpha,
float beta,
const tflite::gpu::HW& destination_size,
id<MTLCommandBuffer> command_buffer,
id<MTLTexture> output_texture) {
mediapipe::Status InternalExecute(id<MTLTexture> input_texture,
const RotatedRect& sub_rect,
bool flip_horizontaly, float alpha,
float beta,
const tflite::gpu::HW& destination_size,
id<MTLCommandBuffer> command_buffer,
id<MTLTexture> output_texture) {
RET_CHECK(command_buffer != nil);
RET_CHECK(output_texture != nil);
// Obtain texture mapping coordinates transformation matrix and copy its
// data to the buffer.
std::array<float, 16> transform_mat;
GetRotatedSubRectToRectTransformMatrix(sub_rect, input_texture.width,
input_texture.height,
flip_horizontaly, &transform_mat_);
std::memcpy(reinterpret_cast<float*>(transform_mat_buffer_.contents),
transform_mat_.data(), sizeof(transform_mat_));
flip_horizontaly, &transform_mat);
id<MTLBuffer> transform_mat_buffer =
[device_ newBufferWithBytes:&transform_mat
length:sizeof(transform_mat)
options:MTLResourceOptionCPUCacheModeDefault];
// Create parameters wrapper.
float parameters[] = {alpha, beta};
@@ -237,7 +243,7 @@ class SubRectExtractorMetal {
[command_encoder setRenderPipelineState:pipeline_state_];
[command_encoder setVertexBuffer:positions_buffer_ offset:0 atIndex:0];
[command_encoder setVertexBuffer:tex_coords_buffer_ offset:0 atIndex:1];
[command_encoder setVertexBuffer:transform_mat_buffer_ offset:0 atIndex:2];
[command_encoder setVertexBuffer:transform_mat_buffer offset:0 atIndex:2];
[command_encoder setFragmentTexture:input_texture atIndex:0];
[command_encoder setFragmentBytes:&parameters
length:sizeof(parameters)
@@ -248,11 +254,11 @@ class SubRectExtractorMetal {
vertexCount:6];
[command_encoder endEncoding];
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
static ::mediapipe::Status MakePipelineState(
id<MTLDevice> device, OutputFormat output_format,
static mediapipe::Status MakePipelineState(
id<MTLDevice> device, OutputFormat output_format, BorderMode border_mode,
id<MTLRenderPipelineState>* pipeline_state) {
RET_CHECK(pipeline_state != nil);
@@ -271,8 +277,25 @@ class SubRectExtractorMetal {
break;
}
std::string shader_lib = absl::StrCat(kShaderLibHeader, output_type_def,
kVertexShader, kFragmentShader);
std::string clamp_def;
switch (border_mode) {
case BorderMode::kReplicate: {
clamp_def = R"(
#define CLAMP_TO_EDGE
)";
break;
}
case BorderMode::kZero: {
clamp_def = R"(
#define CLAMP_TO_ZERO
)";
break;
}
}
std::string shader_lib =
absl::StrCat(kShaderLibHeader, output_type_def, clamp_def,
kVertexShader, kFragmentShader);
NSError* error = nil;
NSString* library_source =
[NSString stringWithUTF8String:shader_lib.c_str()];
@@ -305,27 +328,25 @@ class SubRectExtractorMetal {
RET_CHECK(error == nil) << "Couldn't create a pipeline state"
<< [[error localizedDescription] UTF8String];
return ::mediapipe::OkStatus();
return mediapipe::OkStatus();
}
id<MTLBuffer> positions_buffer_;
id<MTLBuffer> tex_coords_buffer_;
id<MTLBuffer> transform_mat_buffer_;
id<MTLDevice> device_;
id<MTLRenderPipelineState> pipeline_state_;
std::array<float, 16> transform_mat_;
OutputFormat output_format_;
};
class MetalProcessor : public ImageToTensorConverter {
public:
::mediapipe::Status Init(CalculatorContext* cc) {
mediapipe::Status Init(CalculatorContext* cc, BorderMode border_mode) {
metal_helper_ = [[MPPMetalHelper alloc] initWithCalculatorContext:cc];
RET_CHECK(metal_helper_);
ASSIGN_OR_RETURN(extractor_,
SubRectExtractorMetal::Make(metal_helper_.mtlDevice,
OutputFormat::kF32C4));
return ::mediapipe::OkStatus();
ASSIGN_OR_RETURN(extractor_, SubRectExtractorMetal::Make(
metal_helper_.mtlDevice,
OutputFormat::kF32C4, border_mode));
return mediapipe::OkStatus();
}
Size GetImageSize(const Packet& image_packet) override {
@@ -333,11 +354,10 @@ class MetalProcessor : public ImageToTensorConverter {
return {image.width(), image.height()};
}
::mediapipe::StatusOr<Tensor> Convert(const Packet& image_packet,
const RotatedRect& roi,
const Size& output_dims,
float range_min,
float range_max) override {
mediapipe::StatusOr<Tensor> Convert(const Packet& image_packet,
const RotatedRect& roi,
const Size& output_dims, float range_min,
float range_max) override {
const auto& input = image_packet.Get<mediapipe::GpuBuffer>();
if (input.format() != mediapipe::GpuBufferFormat::kBGRA32) {
return InvalidArgumentError(
@@ -369,9 +389,6 @@ class MetalProcessor : public ImageToTensorConverter {
tflite::gpu::HW(output_dims.height, output_dims.width),
command_buffer, buffer_view.buffer()));
[command_buffer commit];
// TODO: consider removing waitUntilCompleted
[command_buffer waitUntilCompleted];
return tensor;
}
}
@@ -383,10 +400,10 @@ class MetalProcessor : public ImageToTensorConverter {
} // namespace
::mediapipe::StatusOr<std::unique_ptr<ImageToTensorConverter>>
CreateMetalConverter(CalculatorContext* cc) {
mediapipe::StatusOr<std::unique_ptr<ImageToTensorConverter>>
CreateMetalConverter(CalculatorContext* cc, BorderMode border_mode) {
auto result = absl::make_unique<MetalProcessor>();
MP_RETURN_IF_ERROR(result->Init(cc));
MP_RETURN_IF_ERROR(result->Init(cc, border_mode));
// Simply "return std::move(result)" failed to build on macOS with bazel.
return std::unique_ptr<ImageToTensorConverter>(std::move(result));
mediapipe/calculators/tensor/image_to_tensor_converter_metal.h
+2 -2
View File
@@ -30,8 +30,8 @@ namespace mediapipe {
// Creates Metal image-to-tensor converter.
// NOTE: [MPPMetalHelper updateContract:...] invocation must precede
// converter creation.
::mediapipe::StatusOr<std::unique_ptr<ImageToTensorConverter>>
CreateMetalConverter(CalculatorContext* cc);
mediapipe::StatusOr<std::unique_ptr<ImageToTensorConverter>>
CreateMetalConverter(CalculatorContext* cc, BorderMode border_mode);
} // namespace mediapipe
mediapipe/calculators/tensor/image_to_tensor_converter_opencv.cc
+22 -9
View File
@@ -35,16 +35,26 @@ namespace {
class OpenCvProcessor : public ImageToTensorConverter {
public:
OpenCvProcessor(BorderMode border_mode) {
switch (border_mode) {
case BorderMode::kReplicate:
border_mode_ = cv::BORDER_REPLICATE;
break;
case BorderMode::kZero:
border_mode_ = cv::BORDER_CONSTANT;
break;
}
}
Size GetImageSize(const Packet& image_packet) override {
const auto& image = image_packet.Get<mediapipe::ImageFrame>();
return {image.Width(), image.Height()};
}
::mediapipe::StatusOr<Tensor> Convert(const Packet& image_packet,
const RotatedRect& roi,
const Size& output_dims,
float range_min,
float range_max) override {
mediapipe::StatusOr<Tensor> Convert(const Packet& image_packet,
const RotatedRect& roi,
const Size& output_dims, float range_min,
float range_max) override {
const auto& input = image_packet.Get<mediapipe::ImageFrame>();
if (input.Format() != mediapipe::ImageFormat::SRGB &&
input.Format() != mediapipe::ImageFormat::SRGBA) {
@@ -84,7 +94,7 @@ class OpenCvProcessor : public ImageToTensorConverter {
cv::warpPerspective(src, transformed, projection_matrix,
cv::Size(dst_width, dst_height),
/*flags=*/cv::INTER_LINEAR,
/*borderMode=*/cv::BORDER_REPLICATE);
/*borderMode=*/border_mode_);
if (transformed.channels() > kNumChannels) {
cv::Mat proper_channels_mat;
@@ -101,16 +111,19 @@ class OpenCvProcessor : public ImageToTensorConverter {
transformed.convertTo(dst, CV_32FC3, transform.scale, transform.offset);
return tensor;
}
private:
enum cv::BorderTypes border_mode_;
};
} // namespace
::mediapipe::StatusOr<std::unique_ptr<ImageToTensorConverter>>
CreateOpenCvConverter(CalculatorContext* cc) {
mediapipe::StatusOr<std::unique_ptr<ImageToTensorConverter>>
CreateOpenCvConverter(CalculatorContext* cc, BorderMode border_mode) {
// Simply "return absl::make_unique<OpenCvProcessor>()" failed to build on
// macOS with bazel.
return std::unique_ptr<ImageToTensorConverter>(
absl::make_unique<OpenCvProcessor>());
absl::make_unique<OpenCvProcessor>(border_mode));
}
} // namespace mediapipe
mediapipe/calculators/tensor/image_to_tensor_converter_opencv.h
+2 -2
View File
@@ -24,8 +24,8 @@
namespace mediapipe {
// Creates OpenCV image-to-tensor converter.
::mediapipe::StatusOr<std::unique_ptr<ImageToTensorConverter>>
CreateOpenCvConverter(CalculatorContext* cc);
mediapipe::StatusOr<std::unique_ptr<ImageToTensorConverter>>
CreateOpenCvConverter(CalculatorContext* cc, BorderMode border_mode);
} // namespace mediapipe

Some files were not shown because too many files have changed in this diff Show More