talksik/mediapipe
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Project import generated by Copybara.

Browse Source 
GitOrigin-RevId: 6e4aff1cc351be3ae4537b677f36d139ee50ce09
This commit is contained in:
MediaPipe Team
2021-03-25 22:09:18 -04:00
committed by chuoling
parent a92cff7a60
commit 7c331ad58b
175 changed files with 4804 additions and 1325 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mediapipe/calculators/audio/BUILD
+12 -12
View File
@@ -128,7 +128,7 @@ cc_library(
"//mediapipe/framework/port:ret_check",
"//mediapipe/util:time_series_util",
"@com_google_absl//absl/strings",
"@eigen_archive//:eigen",
"@eigen_archive//:eigen3",
],
alwayslink = 1,
)
@@ -147,7 +147,7 @@ cc_library(
"//mediapipe/util:time_series_util",
"@com_google_absl//absl/strings",
"@com_google_audio_tools//audio/dsp/mfcc",
"@eigen_archive//:eigen",
"@eigen_archive//:eigen3",
],
alwayslink = 1,
)
@@ -168,7 +168,7 @@ cc_library(
"@com_google_absl//absl/strings",
"@com_google_audio_tools//audio/dsp:resampler",
"@com_google_audio_tools//audio/dsp:resampler_q",
"@eigen_archive//:eigen",
"@eigen_archive//:eigen3",
],
alwayslink = 1,
)
@@ -208,7 +208,7 @@ cc_library(
"@com_google_absl//absl/strings",
"@com_google_audio_tools//audio/dsp:window_functions",
"@com_google_audio_tools//audio/dsp/spectrogram",
"@eigen_archive//:eigen",
"@eigen_archive//:eigen3",
],
alwayslink = 1,
)
@@ -228,7 +228,7 @@ cc_library(
"//mediapipe/framework/port:status",
"//mediapipe/util:time_series_util",
"@com_google_audio_tools//audio/dsp:window_functions",
"@eigen_archive//:eigen",
"@eigen_archive//:eigen3",
],
alwayslink = 1,
)
@@ -242,9 +242,9 @@ cc_test(
"//mediapipe/framework:calculator_runner",
"//mediapipe/framework/deps:file_path",
"//mediapipe/framework/formats:time_series_header_cc_proto",
"//mediapipe/framework/port:commandlineflags",
"//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:parse_text_proto",
"@com_google_absl//absl/flags:flag",
],
)
@@ -261,7 +261,7 @@ cc_test(
"//mediapipe/framework/port:integral_types",
"//mediapipe/framework/port:parse_text_proto",
"//mediapipe/util:time_series_test_util",
"@eigen_archive//:eigen",
"@eigen_archive//:eigen3",
],
)
@@ -276,7 +276,7 @@ cc_test(
"//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:status",
"//mediapipe/util:time_series_test_util",
"@eigen_archive//:eigen",
"@eigen_archive//:eigen3",
],
)
@@ -296,7 +296,7 @@ cc_test(
"//mediapipe/framework/port:status",
"//mediapipe/util:time_series_test_util",
"@com_google_audio_tools//audio/dsp:number_util",
"@eigen_archive//:eigen",
"@eigen_archive//:eigen3",
],
)
@@ -314,7 +314,7 @@ cc_test(
"//mediapipe/framework/port:integral_types",
"//mediapipe/framework/port:status",
"//mediapipe/util:time_series_test_util",
"@eigen_archive//:eigen",
"@eigen_archive//:eigen3",
],
)
@@ -333,7 +333,7 @@ cc_test(
"//mediapipe/framework/port:status",
"//mediapipe/util:time_series_test_util",
"@com_google_audio_tools//audio/dsp:window_functions",
"@eigen_archive//:eigen",
"@eigen_archive//:eigen3",
],
)
@@ -352,6 +352,6 @@ cc_test(
"//mediapipe/framework/tool:validate_type",
"//mediapipe/util:time_series_test_util",
"@com_google_audio_tools//audio/dsp:signal_vector_util",
"@eigen_archive//:eigen",
"@eigen_archive//:eigen3",
],
)
mediapipe/calculators/audio/audio_decoder_calculator_test.cc
+1 -1
View File
@@ -12,10 +12,10 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include "absl/flags/flag.h"
#include "mediapipe/framework/calculator_runner.h"
#include "mediapipe/framework/deps/file_path.h"
#include "mediapipe/framework/formats/time_series_header.pb.h"
#include "mediapipe/framework/port/commandlineflags.h"
#include "mediapipe/framework/port/gmock.h"
#include "mediapipe/framework/port/gtest.h"
#include "mediapipe/framework/port/parse_text_proto.h"
mediapipe/calculators/core/BUILD
+19 -5
View File
@@ -414,7 +414,7 @@ cc_library(
"//mediapipe/framework/api2:node",
"//mediapipe/framework/formats:matrix",
"//mediapipe/framework/port:status",
"@eigen_archive//:eigen",
"@eigen_archive//:eigen3",
],
alwayslink = 1,
)
@@ -430,7 +430,7 @@ cc_library(
"//mediapipe/framework/api2:node",
"//mediapipe/framework/formats:matrix",
"//mediapipe/framework/port:status",
"@eigen_archive//:eigen",
"@eigen_archive//:eigen3",
],
alwayslink = 1,
)
@@ -450,6 +450,20 @@ cc_library(
alwayslink = 1,
)
cc_library(
name = "nonzero_calculator",
srcs = ["nonzero_calculator.cc"],
visibility = [
"//visibility:public",
],
deps = [
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/api2:node",
"//mediapipe/framework/port:ret_check",
],
alwayslink = 1,
)
cc_test(
name = "mux_calculator_test",
srcs = ["mux_calculator_test.cc"],
@@ -776,7 +790,7 @@ cc_test(
"//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:logging",
"//mediapipe/framework/tool:validate_type",
"@eigen_archive//:eigen",
"@eigen_archive//:eigen3",
],
)
@@ -793,7 +807,7 @@ cc_test(
"//mediapipe/framework/port:logging",
"//mediapipe/framework/port:parse_text_proto",
"//mediapipe/framework/tool:validate_type",
"@eigen_archive//:eigen",
"@eigen_archive//:eigen3",
],
)
@@ -1024,7 +1038,7 @@ cc_library(
"//mediapipe/framework/tool:status_util",
"//mediapipe/util:time_series_util",
"@com_google_absl//absl/memory",
"@eigen_archive//:eigen",
"@eigen_archive//:eigen3",
],
alwayslink = 1,
)
mediapipe/calculators/core/flow_limiter_calculator.cc
+1 -1
View File
@@ -57,7 +57,7 @@ namespace mediapipe {
//
// The "ALLOW" stream indicates the transition between accepting frames and
// dropping frames. "ALLOW = true" indicates the start of accepting frames
// including the current timestamp, and "ALLOW = true" indicates the start of
// including the current timestamp, and "ALLOW = false" indicates the start of
// dropping frames including the current timestamp.
//
// FlowLimiterCalculator provides limited support for multiple input streams.
mediapipe/calculators/core/nonzero_calculator.cc
+42
View File
@@ -0,0 +1,42 @@
// Copyright 2021 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/framework/api2/node.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/port/ret_check.h"
namespace mediapipe {
namespace api2 {
// A Calculator that returns 0 if INPUT is 0, and 1 otherwise.
class NonZeroCalculator : public Node {
public:
static constexpr Input<int>::SideFallback kIn{"INPUT"};
static constexpr Output<int> kOut{"OUTPUT"};
MEDIAPIPE_NODE_CONTRACT(kIn, kOut);
absl::Status Process(CalculatorContext* cc) final {
if (!kIn(cc).IsEmpty()) {
auto output = std::make_unique<int>((*kIn(cc) != 0) ? 1 : 0);
kOut(cc).Send(std::move(output));
}
return absl::OkStatus();
}
};
MEDIAPIPE_REGISTER_NODE(NonZeroCalculator);
} // namespace api2
} // namespace mediapipe
mediapipe/calculators/core/packet_resampler_calculator.cc
+508 -200
View File
@@ -87,7 +87,6 @@ absl::Status PacketResamplerCalculator::Open(CalculatorContext* cc) {
flush_last_packet_ = resampler_options.flush_last_packet();
jitter_ = resampler_options.jitter();
jitter_with_reflection_ = resampler_options.jitter_with_reflection();
input_data_id_ = cc->Inputs().GetId("DATA", 0);
if (!input_data_id_.IsValid()) {
@@ -98,11 +97,7 @@ absl::Status PacketResamplerCalculator::Open(CalculatorContext* cc) {
output_data_id_ = cc->Outputs().GetId("", 0);
}
period_count_ = 0;
frame_rate_ = resampler_options.frame_rate();
base_timestamp_ = resampler_options.has_base_timestamp()
? Timestamp(resampler_options.base_timestamp())
: Timestamp::Unset();
start_time_ = resampler_options.has_start_time()
? Timestamp(resampler_options.start_time())
: Timestamp::Min();
@@ -141,30 +136,9 @@ absl::Status PacketResamplerCalculator::Open(CalculatorContext* cc) {
}
}
if (jitter_ != 0.0) {
if (resampler_options.output_header() !=
PacketResamplerCalculatorOptions::NONE) {
LOG(WARNING) << "VideoHeader::frame_rate holds the target value and not "
"the actual value.";
}
if (flush_last_packet_) {
flush_last_packet_ = false;
LOG(WARNING) << "PacketResamplerCalculatorOptions.flush_last_packet is "
"ignored, because we are adding jitter.";
}
const auto& seed = cc->InputSidePackets().Tag("SEED").Get<std::string>();
random_ = CreateSecureRandom(seed);
if (random_ == nullptr) {
return absl::Status(
absl::StatusCode::kInvalidArgument,
"SecureRandom is not available. With \"jitter\" specified, "
"PacketResamplerCalculator processing cannot proceed.");
}
packet_reservoir_random_ = CreateSecureRandom(seed);
}
packet_reservoir_ =
std::make_unique<PacketReservoir>(packet_reservoir_random_.get());
return absl::OkStatus();
strategy_ = GetSamplingStrategy(resampler_options);
return strategy_->Open(cc);
}
absl::Status PacketResamplerCalculator::Process(CalculatorContext* cc) {
@@ -177,171 +151,13 @@ absl::Status PacketResamplerCalculator::Process(CalculatorContext* cc) {
return absl::OkStatus();
}
}
if (jitter_ != 0.0 && random_ != nullptr) {
// Packet reservior is used to make sure there's an output for every period,
// e.g. partial period at the end of the stream.
if (packet_reservoir_->IsEnabled() &&
(first_timestamp_ == Timestamp::Unset() ||
(cc->InputTimestamp() - next_output_timestamp_min_).Value() >= 0)) {
auto curr_packet = cc->Inputs().Get(input_data_id_).Value();
packet_reservoir_->AddSample(curr_packet);
}
MP_RETURN_IF_ERROR(ProcessWithJitter(cc));
} else {
MP_RETURN_IF_ERROR(ProcessWithoutJitter(cc));
if (absl::Status status = strategy_->Process(cc); !status.ok()) {
return status; // Avoid MP_RETURN_IF_ERROR macro for external release.
}
last_packet_ = cc->Inputs().Get(input_data_id_).Value();
return absl::OkStatus();
}
void PacketResamplerCalculator::InitializeNextOutputTimestampWithJitter() {
next_output_timestamp_min_ = first_timestamp_;
if (jitter_with_reflection_) {
next_output_timestamp_ =
first_timestamp_ + random_->UnbiasedUniform64(frame_time_usec_);
return;
}
next_output_timestamp_ =
first_timestamp_ + frame_time_usec_ * random_->RandFloat();
}
void PacketResamplerCalculator::UpdateNextOutputTimestampWithJitter() {
packet_reservoir_->Clear();
if (jitter_with_reflection_) {
next_output_timestamp_min_ += frame_time_usec_;
Timestamp next_output_timestamp_max_ =
next_output_timestamp_min_ + frame_time_usec_;
next_output_timestamp_ += frame_time_usec_ +
random_->UnbiasedUniform64(2 * jitter_usec_ + 1) -
jitter_usec_;
next_output_timestamp_ = Timestamp(ReflectBetween(
next_output_timestamp_.Value(), next_output_timestamp_min_.Value(),
next_output_timestamp_max_.Value()));
CHECK_GE(next_output_timestamp_, next_output_timestamp_min_);
CHECK_LT(next_output_timestamp_, next_output_timestamp_max_);
return;
}
packet_reservoir_->Disable();
next_output_timestamp_ +=
frame_time_usec_ *
((1.0 - jitter_) + 2.0 * jitter_ * random_->RandFloat());
}
absl::Status PacketResamplerCalculator::ProcessWithJitter(
CalculatorContext* cc) {
RET_CHECK_GT(cc->InputTimestamp(), Timestamp::PreStream());
RET_CHECK_NE(jitter_, 0.0);
if (first_timestamp_ == Timestamp::Unset()) {
first_timestamp_ = cc->InputTimestamp();
InitializeNextOutputTimestampWithJitter();
if (first_timestamp_ == next_output_timestamp_) {
OutputWithinLimits(
cc,
cc->Inputs().Get(input_data_id_).Value().At(next_output_timestamp_));
UpdateNextOutputTimestampWithJitter();
}
return absl::OkStatus();
}
if (frame_time_usec_ <
(cc->InputTimestamp() - last_packet_.Timestamp()).Value()) {
LOG_FIRST_N(WARNING, 2)
<< "Adding jitter is not very useful when upsampling.";
}
while (true) {
const int64 last_diff =
(next_output_timestamp_ - last_packet_.Timestamp()).Value();
RET_CHECK_GT(last_diff, 0);
const int64 curr_diff =
(next_output_timestamp_ - cc->InputTimestamp()).Value();
if (curr_diff > 0) {
break;
}
OutputWithinLimits(cc, (std::abs(curr_diff) > last_diff
? last_packet_
: cc->Inputs().Get(input_data_id_).Value())
.At(next_output_timestamp_));
UpdateNextOutputTimestampWithJitter();
// From now on every time a packet is emitted the timestamp of the next
// packet becomes known; that timestamp is stored in next_output_timestamp_.
// The only exception to this rule is the packet emitted from Close() which
// can only happen when jitter_with_reflection is enabled but in this case
// next_output_timestamp_min_ is a non-decreasing lower bound of any
// subsequent packet.
const Timestamp timestamp_bound = jitter_with_reflection_
? next_output_timestamp_min_
: next_output_timestamp_;
cc->Outputs().Get(output_data_id_).SetNextTimestampBound(timestamp_bound);
}
return absl::OkStatus();
}
absl::Status PacketResamplerCalculator::ProcessWithoutJitter(
CalculatorContext* cc) {
RET_CHECK_GT(cc->InputTimestamp(), Timestamp::PreStream());
RET_CHECK_EQ(jitter_, 0.0);
if (first_timestamp_ == Timestamp::Unset()) {
// This is the first packet, initialize the first_timestamp_.
if (base_timestamp_ == Timestamp::Unset()) {
// Initialize first_timestamp_ with exactly the first packet timestamp.
first_timestamp_ = cc->InputTimestamp();
} else {
// Initialize first_timestamp_ with the first packet timestamp
// aligned to the base_timestamp_.
int64 first_index = MathUtil::SafeRound<int64, double>(
(cc->InputTimestamp() - base_timestamp_).Seconds() * frame_rate_);
first_timestamp_ =
base_timestamp_ + TimestampDiffFromSeconds(first_index / frame_rate_);
}
if (cc->Outputs().UsesTags() && cc->Outputs().HasTag("VIDEO_HEADER")) {
cc->Outputs()
.Tag("VIDEO_HEADER")
.Add(new VideoHeader(video_header_), Timestamp::PreStream());
}
}
const Timestamp received_timestamp = cc->InputTimestamp();
const int64 received_timestamp_idx =
TimestampToPeriodIndex(received_timestamp);
// Only consider the received packet if it belongs to the current period
// (== period_count_) or to a newer one (> period_count_).
if (received_timestamp_idx >= period_count_) {
// Fill the empty periods until we are in the same index as the received
// packet.
while (received_timestamp_idx > period_count_) {
OutputWithinLimits(
cc, last_packet_.At(PeriodIndexToTimestamp(period_count_)));
++period_count_;
}
// Now, if the received packet has a timestamp larger than the middle of
// the current period, we can send a packet without waiting. We send the
// one closer to the middle.
Timestamp target_timestamp = PeriodIndexToTimestamp(period_count_);
if (received_timestamp >= target_timestamp) {
bool have_last_packet = (last_packet_.Timestamp() != Timestamp::Unset());
bool send_current =
!have_last_packet || (received_timestamp - target_timestamp <=
target_timestamp - last_packet_.Timestamp());
if (send_current) {
OutputWithinLimits(
cc, cc->Inputs().Get(input_data_id_).Value().At(target_timestamp));
} else {
OutputWithinLimits(cc, last_packet_.At(target_timestamp));
}
++period_count_;
}
// TODO: Add a mechanism to the framework to allow these packets
// to be output earlier (without waiting for a much later packet to
// arrive)
// Update the bound for the next packet.
cc->Outputs()
.Get(output_data_id_)
.SetNextTimestampBound(PeriodIndexToTimestamp(period_count_));
}
return absl::OkStatus();
}
@@ -349,17 +165,34 @@ absl::Status PacketResamplerCalculator::Close(CalculatorContext* cc) {
if (!cc->GraphStatus().ok()) {
return absl::OkStatus();
}
// Emit the last packet received if we have at least one packet, but
// haven't sent anything for its period.
if (first_timestamp_ != Timestamp::Unset() && flush_last_packet_ &&
TimestampToPeriodIndex(last_packet_.Timestamp()) == period_count_) {
OutputWithinLimits(cc,
last_packet_.At(PeriodIndexToTimestamp(period_count_)));
return strategy_->Close(cc);
}
std::unique_ptr<PacketResamplerStrategy>
PacketResamplerCalculator::GetSamplingStrategy(
const PacketResamplerCalculatorOptions& options) {
if (options.reproducible_sampling()) {
if (!options.jitter_with_reflection()) {
LOG(WARNING)
<< "reproducible_sampling enabled w/ jitter_with_reflection "
"disabled. "
<< "reproducible_sampling always uses jitter with reflection, "
<< "Ignoring jitter_with_reflection setting.";
}
return absl::make_unique<ReproducibleJitterWithReflectionStrategy>(this);
}
if (!packet_reservoir_->IsEmpty()) {
OutputWithinLimits(cc, packet_reservoir_->GetSample());
if (options.jitter() == 0) {
return absl::make_unique<NoJitterStrategy>(this);
}
return absl::OkStatus();
if (options.jitter_with_reflection()) {
return absl::make_unique<LegacyJitterWithReflectionStrategy>(this);
}
// With jitter and no reflection.
return absl::make_unique<JitterWithoutReflectionStrategy>(this);
}
Timestamp PacketResamplerCalculator::PeriodIndexToTimestamp(int64 index) const {
@@ -385,4 +218,479 @@ void PacketResamplerCalculator::OutputWithinLimits(CalculatorContext* cc,
}
}
absl::Status LegacyJitterWithReflectionStrategy::Open(CalculatorContext* cc) {
const auto resampler_options =
tool::RetrieveOptions(cc->Options<PacketResamplerCalculatorOptions>(),
cc->InputSidePackets(), "OPTIONS");
if (resampler_options.output_header() !=
PacketResamplerCalculatorOptions::NONE) {
LOG(WARNING) << "VideoHeader::frame_rate holds the target value and not "
"the actual value.";
}
if (calculator_->flush_last_packet_) {
LOG(WARNING) << "PacketResamplerCalculatorOptions.flush_last_packet is "
"ignored, because we are adding jitter.";
}
const auto& seed = cc->InputSidePackets().Tag("SEED").Get<std::string>();
random_ = CreateSecureRandom(seed);
if (random_ == nullptr) {
return absl::InvalidArgumentError(
"SecureRandom is not available. With \"jitter\" specified, "
"PacketResamplerCalculator processing cannot proceed.");
}
packet_reservoir_random_ = CreateSecureRandom(seed);
packet_reservoir_ =
std::make_unique<PacketReservoir>(packet_reservoir_random_.get());
return absl::OkStatus();
}
absl::Status LegacyJitterWithReflectionStrategy::Close(CalculatorContext* cc) {
if (!packet_reservoir_->IsEmpty()) {
LOG(INFO) << "Emitting pack from reservoir.";
calculator_->OutputWithinLimits(cc, packet_reservoir_->GetSample());
}
return absl::OkStatus();
}
absl::Status LegacyJitterWithReflectionStrategy::Process(
CalculatorContext* cc) {
RET_CHECK_GT(cc->InputTimestamp(), Timestamp::PreStream());
if (packet_reservoir_->IsEnabled() &&
(first_timestamp_ == Timestamp::Unset() ||
(cc->InputTimestamp() - next_output_timestamp_min_).Value() >= 0)) {
auto curr_packet = cc->Inputs().Get(calculator_->input_data_id_).Value();
packet_reservoir_->AddSample(curr_packet);
}
if (first_timestamp_ == Timestamp::Unset()) {
first_timestamp_ = cc->InputTimestamp();
InitializeNextOutputTimestampWithJitter();
if (first_timestamp_ == next_output_timestamp_) {
calculator_->OutputWithinLimits(cc, cc->Inputs()
.Get(calculator_->input_data_id_)
.Value()
.At(next_output_timestamp_));
UpdateNextOutputTimestampWithJitter();
}
return absl::OkStatus();
}
if (calculator_->frame_time_usec_ <
(cc->InputTimestamp() - calculator_->last_packet_.Timestamp()).Value()) {
LOG_FIRST_N(WARNING, 2)
<< "Adding jitter is not very useful when upsampling.";
}
while (true) {
const int64 last_diff =
(next_output_timestamp_ - calculator_->last_packet_.Timestamp())
.Value();
RET_CHECK_GT(last_diff, 0);
const int64 curr_diff =
(next_output_timestamp_ - cc->InputTimestamp()).Value();
if (curr_diff > 0) {
break;
}
calculator_->OutputWithinLimits(
cc, (std::abs(curr_diff) > last_diff
? calculator_->last_packet_
: cc->Inputs().Get(calculator_->input_data_id_).Value())
.At(next_output_timestamp_));
UpdateNextOutputTimestampWithJitter();
// From now on every time a packet is emitted the timestamp of the next
// packet becomes known; that timestamp is stored in next_output_timestamp_.
// The only exception to this rule is the packet emitted from Close() which
// can only happen when jitter_with_reflection is enabled but in this case
// next_output_timestamp_min_ is a non-decreasing lower bound of any
// subsequent packet.
const Timestamp timestamp_bound = next_output_timestamp_min_;
cc->Outputs()
.Get(calculator_->output_data_id_)
.SetNextTimestampBound(timestamp_bound);
}
return absl::OkStatus();
}
void LegacyJitterWithReflectionStrategy::
InitializeNextOutputTimestampWithJitter() {
next_output_timestamp_min_ = first_timestamp_;
next_output_timestamp_ =
first_timestamp_ +
random_->UnbiasedUniform64(calculator_->frame_time_usec_);
}
void LegacyJitterWithReflectionStrategy::UpdateNextOutputTimestampWithJitter() {
packet_reservoir_->Clear();
next_output_timestamp_min_ += calculator_->frame_time_usec_;
Timestamp next_output_timestamp_max_ =
next_output_timestamp_min_ + calculator_->frame_time_usec_;
next_output_timestamp_ +=
calculator_->frame_time_usec_ +
random_->UnbiasedUniform64(2 * calculator_->jitter_usec_ + 1) -
calculator_->jitter_usec_;
next_output_timestamp_ = Timestamp(ReflectBetween(
next_output_timestamp_.Value(), next_output_timestamp_min_.Value(),
next_output_timestamp_max_.Value()));
CHECK_GE(next_output_timestamp_, next_output_timestamp_min_);
CHECK_LT(next_output_timestamp_, next_output_timestamp_max_);
}
absl::Status ReproducibleJitterWithReflectionStrategy::Open(
CalculatorContext* cc) {
const auto resampler_options =
tool::RetrieveOptions(cc->Options<PacketResamplerCalculatorOptions>(),
cc->InputSidePackets(), "OPTIONS");
if (resampler_options.output_header() !=
PacketResamplerCalculatorOptions::NONE) {
LOG(WARNING) << "VideoHeader::frame_rate holds the target value and not "
"the actual value.";
}
if (calculator_->flush_last_packet_) {
LOG(WARNING) << "PacketResamplerCalculatorOptions.flush_last_packet is "
"ignored, because we are adding jitter.";
}
const auto& seed = cc->InputSidePackets().Tag("SEED").Get<std::string>();
random_ = CreateSecureRandom(seed);
if (random_ == nullptr) {
return absl::InvalidArgumentError(
"SecureRandom is not available. With \"jitter\" specified, "
"PacketResamplerCalculator processing cannot proceed.");
}
return absl::OkStatus();
}
absl::Status ReproducibleJitterWithReflectionStrategy::Close(
CalculatorContext* cc) {
// If last packet is non-empty and a packet hasn't been emitted for this
// period, emit the last packet.
if (!calculator_->last_packet_.IsEmpty() && !packet_emitted_this_period_) {
calculator_->OutputWithinLimits(
cc, calculator_->last_packet_.At(next_output_timestamp_));
}
return absl::OkStatus();
}
absl::Status ReproducibleJitterWithReflectionStrategy::Process(
CalculatorContext* cc) {
RET_CHECK_GT(cc->InputTimestamp(), Timestamp::PreStream());
Packet current_packet = cc->Inputs().Get(calculator_->input_data_id_).Value();
if (calculator_->last_packet_.IsEmpty()) {
// last_packet is empty, this is the first packet of the stream.
InitializeNextOutputTimestamp(current_packet.Timestamp());
// If next_output_timestamp_ happens to fall before current_packet, emit
// current packet. Only a single packet can be emitted at the beginning
// of the stream.
if (next_output_timestamp_ < current_packet.Timestamp()) {
calculator_->OutputWithinLimits(
cc, current_packet.At(next_output_timestamp_));
packet_emitted_this_period_ = true;
}
return absl::OkStatus();
}
// Last packet is set, so we are mid-stream.
if (calculator_->frame_time_usec_ <
(current_packet.Timestamp() - calculator_->last_packet_.Timestamp())
.Value()) {
// Note, if the stream is upsampling, this could lead to the same packet
// being emitted twice. Upsampling and jitter doesn't make much sense
// but does technically work.
LOG_FIRST_N(WARNING, 2)
<< "Adding jitter is not very useful when upsampling.";
}
// Since we may be upsampling, we need to iteratively advance the
// next_output_timestamp_ one period at a time until it reaches the period
// current_packet is in. During this process, last_packet and/or
// current_packet may be repeatly emitted.
UpdateNextOutputTimestamp(current_packet.Timestamp());
while (!packet_emitted_this_period_ &&
next_output_timestamp_ <= current_packet.Timestamp()) {
// last_packet < next_output_timestamp_ <= current_packet,
// so emit the closest packet.
Packet packet_to_emit =
current_packet.Timestamp() - next_output_timestamp_ <
next_output_timestamp_ - calculator_->last_packet_.Timestamp()
? current_packet
: calculator_->last_packet_;
calculator_->OutputWithinLimits(cc,
packet_to_emit.At(next_output_timestamp_));
packet_emitted_this_period_ = true;
// If we are upsampling, packet_emitted_this_period_ can be reset by
// the following UpdateNext and the loop will iterate.
UpdateNextOutputTimestamp(current_packet.Timestamp());
}
// Set the bounds on the output stream. Note, if we emitted a packet
// above, it will already be set at next_output_timestamp_ + 1, in which
// case we have to skip setting it.
if (cc->Outputs().Get(calculator_->output_data_id_).NextTimestampBound() <
next_output_timestamp_) {
cc->Outputs()
.Get(calculator_->output_data_id_)
.SetNextTimestampBound(next_output_timestamp_);
}
return absl::OkStatus();
}
void ReproducibleJitterWithReflectionStrategy::InitializeNextOutputTimestamp(
Timestamp current_timestamp) {
if (next_output_timestamp_min_ != Timestamp::Unset()) {
return;
}
next_output_timestamp_min_ = Timestamp(0);
next_output_timestamp_ =
Timestamp(GetNextRandom(calculator_->frame_time_usec_));
// While the current timestamp is ahead of the max (i.e. min + frame_time),
// fast-forward.
while (current_timestamp >=
next_output_timestamp_min_ + calculator_->frame_time_usec_) {
packet_emitted_this_period_ = true; // Force update...
UpdateNextOutputTimestamp(current_timestamp);
}
}
void ReproducibleJitterWithReflectionStrategy::UpdateNextOutputTimestamp(
Timestamp current_timestamp) {
if (packet_emitted_this_period_ &&
current_timestamp >=
next_output_timestamp_min_ + calculator_->frame_time_usec_) {
next_output_timestamp_min_ += calculator_->frame_time_usec_;
Timestamp next_output_timestamp_max_ =
next_output_timestamp_min_ + calculator_->frame_time_usec_;
next_output_timestamp_ += calculator_->frame_time_usec_ +
GetNextRandom(2 * calculator_->jitter_usec_ + 1) -
calculator_->jitter_usec_;
next_output_timestamp_ = Timestamp(ReflectBetween(
next_output_timestamp_.Value(), next_output_timestamp_min_.Value(),
next_output_timestamp_max_.Value()));
packet_emitted_this_period_ = false;
}
}
absl::Status JitterWithoutReflectionStrategy::Open(CalculatorContext* cc) {
const auto resampler_options =
tool::RetrieveOptions(cc->Options<PacketResamplerCalculatorOptions>(),
cc->InputSidePackets(), "OPTIONS");
if (resampler_options.output_header() !=
PacketResamplerCalculatorOptions::NONE) {
LOG(WARNING) << "VideoHeader::frame_rate holds the target value and not "
"the actual value.";
}
if (calculator_->flush_last_packet_) {
LOG(WARNING) << "PacketResamplerCalculatorOptions.flush_last_packet is "
"ignored, because we are adding jitter.";
}
const auto& seed = cc->InputSidePackets().Tag("SEED").Get<std::string>();
random_ = CreateSecureRandom(seed);
if (random_ == nullptr) {
return absl::InvalidArgumentError(
"SecureRandom is not available. With \"jitter\" specified, "
"PacketResamplerCalculator processing cannot proceed.");
}
packet_reservoir_random_ = CreateSecureRandom(seed);
packet_reservoir_ =
absl::make_unique<PacketReservoir>(packet_reservoir_random_.get());
return absl::OkStatus();
}
absl::Status JitterWithoutReflectionStrategy::Close(CalculatorContext* cc) {
if (!packet_reservoir_->IsEmpty()) {
calculator_->OutputWithinLimits(cc, packet_reservoir_->GetSample());
}
return absl::OkStatus();
}
absl::Status JitterWithoutReflectionStrategy::Process(CalculatorContext* cc) {
RET_CHECK_GT(cc->InputTimestamp(), Timestamp::PreStream());
// Packet reservior is used to make sure there's an output for every period,
// e.g. partial period at the end of the stream.
if (packet_reservoir_->IsEnabled() &&
(calculator_->first_timestamp_ == Timestamp::Unset() ||
(cc->InputTimestamp() - next_output_timestamp_min_).Value() >= 0)) {
auto curr_packet = cc->Inputs().Get(calculator_->input_data_id_).Value();
packet_reservoir_->AddSample(curr_packet);
}
if (calculator_->first_timestamp_ == Timestamp::Unset()) {
calculator_->first_timestamp_ = cc->InputTimestamp();
InitializeNextOutputTimestamp();
if (calculator_->first_timestamp_ == next_output_timestamp_) {
calculator_->OutputWithinLimits(cc, cc->Inputs()
.Get(calculator_->input_data_id_)
.Value()
.At(next_output_timestamp_));
UpdateNextOutputTimestamp();
}
return absl::OkStatus();
}
if (calculator_->frame_time_usec_ <
(cc->InputTimestamp() - calculator_->last_packet_.Timestamp()).Value()) {
LOG_FIRST_N(WARNING, 2)
<< "Adding jitter is not very useful when upsampling.";
}
while (true) {
const int64 last_diff =
(next_output_timestamp_ - calculator_->last_packet_.Timestamp())
.Value();
RET_CHECK_GT(last_diff, 0);
const int64 curr_diff =
(next_output_timestamp_ - cc->InputTimestamp()).Value();
if (curr_diff > 0) {
break;
}
calculator_->OutputWithinLimits(
cc, (std::abs(curr_diff) > last_diff
? calculator_->last_packet_
: cc->Inputs().Get(calculator_->input_data_id_).Value())
.At(next_output_timestamp_));
UpdateNextOutputTimestamp();
cc->Outputs()
.Get(calculator_->output_data_id_)
.SetNextTimestampBound(next_output_timestamp_);
}
return absl::OkStatus();
}
void JitterWithoutReflectionStrategy::InitializeNextOutputTimestamp() {
next_output_timestamp_min_ = calculator_->first_timestamp_;
next_output_timestamp_ = calculator_->first_timestamp_ +
calculator_->frame_time_usec_ * random_->RandFloat();
}
void JitterWithoutReflectionStrategy::UpdateNextOutputTimestamp() {
packet_reservoir_->Clear();
packet_reservoir_->Disable();
next_output_timestamp_ += calculator_->frame_time_usec_ *
((1.0 - calculator_->jitter_) +
2.0 * calculator_->jitter_ * random_->RandFloat());
}
absl::Status NoJitterStrategy::Open(CalculatorContext* cc) {
const auto resampler_options =
tool::RetrieveOptions(cc->Options<PacketResamplerCalculatorOptions>(),
cc->InputSidePackets(), "OPTIONS");
base_timestamp_ = resampler_options.has_base_timestamp()
? Timestamp(resampler_options.base_timestamp())
: Timestamp::Unset();
period_count_ = 0;
return absl::OkStatus();
}
absl::Status NoJitterStrategy::Close(CalculatorContext* cc) {
// Emit the last packet received if we have at least one packet, but
// haven't sent anything for its period.
if (calculator_->first_timestamp_ != Timestamp::Unset() &&
calculator_->flush_last_packet_ &&
calculator_->TimestampToPeriodIndex(
calculator_->last_packet_.Timestamp()) == period_count_) {
calculator_->OutputWithinLimits(
cc, calculator_->last_packet_.At(
calculator_->PeriodIndexToTimestamp(period_count_)));
}
return absl::OkStatus();
}
absl::Status NoJitterStrategy::Process(CalculatorContext* cc) {
RET_CHECK_GT(cc->InputTimestamp(), Timestamp::PreStream());
if (calculator_->first_timestamp_ == Timestamp::Unset()) {
// This is the first packet, initialize the first_timestamp_.
if (base_timestamp_ == Timestamp::Unset()) {
// Initialize first_timestamp_ with exactly the first packet timestamp.
calculator_->first_timestamp_ = cc->InputTimestamp();
} else {
// Initialize first_timestamp_ with the first packet timestamp
// aligned to the base_timestamp_.
int64 first_index = MathUtil::SafeRound<int64, double>(
(cc->InputTimestamp() - base_timestamp_).Seconds() *
calculator_->frame_rate_);
calculator_->first_timestamp_ =
base_timestamp_ +
TimestampDiffFromSeconds(first_index / calculator_->frame_rate_);
}
if (cc->Outputs().UsesTags() && cc->Outputs().HasTag("VIDEO_HEADER")) {
cc->Outputs()
.Tag("VIDEO_HEADER")
.Add(new VideoHeader(calculator_->video_header_),
Timestamp::PreStream());
}
}
const Timestamp received_timestamp = cc->InputTimestamp();
const int64 received_timestamp_idx =
calculator_->TimestampToPeriodIndex(received_timestamp);
// Only consider the received packet if it belongs to the current period
// (== period_count_) or to a newer one (> period_count_).
if (received_timestamp_idx >= period_count_) {
// Fill the empty periods until we are in the same index as the received
// packet.
while (received_timestamp_idx > period_count_) {
calculator_->OutputWithinLimits(
cc, calculator_->last_packet_.At(
calculator_->PeriodIndexToTimestamp(period_count_)));
++period_count_;
}
// Now, if the received packet has a timestamp larger than the middle of
// the current period, we can send a packet without waiting. We send the
// one closer to the middle.
Timestamp target_timestamp =
calculator_->PeriodIndexToTimestamp(period_count_);
if (received_timestamp >= target_timestamp) {
bool have_last_packet =
(calculator_->last_packet_.Timestamp() != Timestamp::Unset());
bool send_current =
!have_last_packet ||
(received_timestamp - target_timestamp <=
target_timestamp - calculator_->last_packet_.Timestamp());
if (send_current) {
calculator_->OutputWithinLimits(cc,
cc->Inputs()
.Get(calculator_->input_data_id_)
.Value()
.At(target_timestamp));
} else {
calculator_->OutputWithinLimits(
cc, calculator_->last_packet_.At(target_timestamp));
}
++period_count_;
}
// TODO: Add a mechanism to the framework to allow these packets
// to be output earlier (without waiting for a much later packet to
// arrive)
// Update the bound for the next packet.
cc->Outputs()
.Get(calculator_->output_data_id_)
.SetNextTimestampBound(
calculator_->PeriodIndexToTimestamp(period_count_));
}
return absl::OkStatus();
}
} // namespace mediapipe
mediapipe/calculators/core/packet_resampler_calculator.h
+223 -36
View File
@@ -55,7 +55,7 @@ class PacketReservoir {
// correspond to timestamp t.
// - The next packet is chosen randomly (uniform distribution) among frames
// that correspond to [t+(1-jitter)/frame_rate, t+(1+jitter)/frame_rate].
// - if jitter_with_reflection_ is true, the timestamp will be reflected
// - if jitter_with_reflection is true, the timestamp will be reflected
// against the boundaries of [t_0 + (k-1)/frame_rate, t_0 + k/frame_rate)
// so that its marginal distribution is uniform within this interval.
// In the formula, t_0 is the timestamp of the first sampled
@@ -66,6 +66,17 @@ class PacketReservoir {
// the resampling. For Cloud ML Video Intelligence API, the hash of the
// input video should serve this purpose. For YouTube, either video ID or
// content hex ID of the input video should do.
// - If reproducible_samping is true, care is taken to allow reproducible
// "mid-stream" sampling. The calculator can be executed on a stream that
// doesn't start at the first period. For instance, if the calculator
// is run on a 10 second stream it will produce the same set of samples
// as two runs of the calculator, the first with 3 seconds of input starting
// at time 0 and the second with 7 seconds of input starting at time +3s.
// - In order to guarantee the exact same samples, 1) the inputs must be
// aligned with the sampling period. For instance, if the sampling rate
// is 2 frames per second, streams should be aligned on 0.5 second
// boundaries, and 2) the stream must include at least one extra packet
// before and after the second aligned sampling period.
//
// If jitter_ is not specified:
// - The first packet defines the first_timestamp of the output stream,
@@ -105,19 +116,6 @@ class PacketResamplerCalculator : public CalculatorBase {
absl::Status Close(CalculatorContext* cc) override;
absl::Status Process(CalculatorContext* cc) override;
private:
// Calculates the first sampled timestamp that incorporates a jittering
// offset.
void InitializeNextOutputTimestampWithJitter();
// Calculates the next sampled timestamp that incorporates a jittering offset.
void UpdateNextOutputTimestampWithJitter();
// Logic for Process() when jitter_ != 0.0.
absl::Status ProcessWithJitter(CalculatorContext* cc);
// Logic for Process() when jitter_ == 0.0.
absl::Status ProcessWithoutJitter(CalculatorContext* cc);
// Given the current count of periods that have passed, this returns
// the next valid timestamp of the middle point of the next period:
// if count is 0, it returns the first_timestamp_.
@@ -141,6 +139,16 @@ class PacketResamplerCalculator : public CalculatorBase {
// Outputs a packet if it is in range (start_time_, end_time_).
void OutputWithinLimits(CalculatorContext* cc, const Packet& packet) const;
protected:
// Returns Sampling Strategy to use.
//
// Virtual to allow injection of testing strategies.
virtual std::unique_ptr<class PacketResamplerStrategy> GetSamplingStrategy(
const mediapipe::PacketResamplerCalculatorOptions& options);
private:
std::unique_ptr<class PacketResamplerStrategy> strategy_;
// The timestamp of the first packet received.
Timestamp first_timestamp_;
@@ -150,14 +158,6 @@ class PacketResamplerCalculator : public CalculatorBase {
// Inverse of frame_rate_.
int64 frame_time_usec_;
// Number of periods that have passed (= #packets sent to the output).
//
// Can only be used if jitter_ equals zero.
int64 period_count_;
// The last packet that was received.
Packet last_packet_;
VideoHeader video_header_;
// The "DATA" input stream.
CollectionItemId input_data_id_;
@@ -165,23 +165,15 @@ class PacketResamplerCalculator : public CalculatorBase {
CollectionItemId output_data_id_;
// Indicator whether to flush last packet even if its timestamp is greater
// than the final stream timestamp. Set to false when jitter_ is non-zero.
// than the final stream timestamp.
bool flush_last_packet_;
// Jitter-related variables.
std::unique_ptr<RandomBase> random_;
double jitter_ = 0.0;
bool jitter_with_reflection_;
int64 jitter_usec_;
Timestamp next_output_timestamp_;
// If jittering_with_reflection_ is true, next_output_timestamp_ will be
// kept within the interval
// [next_output_timestamp_min_, next_output_timestamp_min_ + frame_time_usec_)
Timestamp next_output_timestamp_min_;
// If specified, output timestamps are aligned with base_timestamp.
// Otherwise, they are aligned with the first input timestamp.
Timestamp base_timestamp_;
int64 jitter_usec_;
// The last packet that was received.
Packet last_packet_;
// If specified, only outputs at/after start_time are included.
Timestamp start_time_;
@@ -191,15 +183,210 @@ class PacketResamplerCalculator : public CalculatorBase {
// If set, the output timestamps nearest to start_time and end_time
// are included in the output, even if the nearest timestamp is not
// between start_time and end_time.W
// between start_time and end_time.
bool round_limits_;
// Allow strategies access to all internal calculator state.
//
// The calculator and strategies are intimiately tied together so this should
// not break encapsulation.
friend class LegacyJitterWithReflectionStrategy;
friend class ReproducibleJitterWithReflectionStrategy;
friend class JitterWithoutReflectionStrategy;
friend class NoJitterStrategy;
};
// Abstract class encapsulating sampling stategy.
//
// These are used solely by PacketResamplerCalculator, but are exposed here
// to facilitate tests.
class PacketResamplerStrategy {
public:
PacketResamplerStrategy(PacketResamplerCalculator* calculator)
: calculator_(calculator) {}
virtual ~PacketResamplerStrategy() = default;
// Delegate for CalculatorBase::Open. See CalculatorBase for relevant
// implementation considerations.
virtual absl::Status Open(CalculatorContext* cc) = 0;
// Delegate for CalculatorBase::Close. See CalculatorBase for relevant
// implementation considerations.
virtual absl::Status Close(CalculatorContext* cc) = 0;
// Delegate for CalculatorBase::Process. See CalculatorBase for relevant
// implementation considerations.
virtual absl::Status Process(CalculatorContext* cc) = 0;
protected:
// Calculator running strategy.
PacketResamplerCalculator* calculator_;
};
// Strategy that applies Jitter with reflection based sampling.
//
// Used by PacketResamplerCalculator when both Jitter and reflection are
// enabled.
//
// This applies the legacy jitter with reflection which doesn't allow
// for reproducibility of sampling when starting mid-stream. This is maintained
// for backward compatibility.
class LegacyJitterWithReflectionStrategy : public PacketResamplerStrategy {
public:
LegacyJitterWithReflectionStrategy(PacketResamplerCalculator* calculator)
: PacketResamplerStrategy(calculator) {}
absl::Status Open(CalculatorContext* cc) override;
absl::Status Close(CalculatorContext* cc) override;
absl::Status Process(CalculatorContext* cc) override;
private:
void InitializeNextOutputTimestampWithJitter();
void UpdateNextOutputTimestampWithJitter();
// Jitter-related variables.
std::unique_ptr<RandomBase> random_;
// The timestamp of the first packet received.
Timestamp first_timestamp_;
// Next packet to be emitted. Since packets may not align perfectly with
// next_output_timestamp_, the closest packet will be emitted.
Timestamp next_output_timestamp_;
// Lower bound for next timestamp.
//
// next_output_timestamp_ will be kept within the interval
// [next_output_timestamp_min_, next_output_timestamp_min_ + frame_time_usec_)
Timestamp next_output_timestamp_min_ = Timestamp::Unset();
// packet reservior used for sampling random packet out of partial
// period when jitter is enabled
std::unique_ptr<PacketReservoir> packet_reservoir_;
// random number generator used in packet_reservior_.
std::unique_ptr<RandomBase> packet_reservoir_random_;
};
// Strategy that applies reproducible jitter with reflection based sampling.
//
// Used by PacketResamplerCalculator when both Jitter and reflection are
// enabled.
class ReproducibleJitterWithReflectionStrategy
: public PacketResamplerStrategy {
public:
ReproducibleJitterWithReflectionStrategy(
PacketResamplerCalculator* calculator)
: PacketResamplerStrategy(calculator) {}
absl::Status Open(CalculatorContext* cc) override;
absl::Status Close(CalculatorContext* cc) override;
absl::Status Process(CalculatorContext* cc) override;
protected:
// Returns next random in range (0,n].
//
// Exposed as virtual function for testing Jitter with reflection.
// This is the only way random_ is accessed.
virtual uint64 GetNextRandom(uint64 n) {
return random_->UnbiasedUniform64(n);
}
private:
// Initializes Jitter with reflection.
//
// This will fast-forward to the period containing current_timestamp.
// next_output_timestamp_ is guarnateed to be current_timestamp's period
// and packet_emitted_this_period_ will be set to false.
void InitializeNextOutputTimestamp(Timestamp current_timestamp);
// Potentially advances next_output_timestamp_ a single period.
//
// next_output_timestamp_ will only be advanced if packet_emitted_this_period_
// is false. next_output_timestamp_ will never be advanced beyond
// current_timestamp's period.
//
// However, next_output_timestamp_ could fall before current_timestamp's
// period since only a single period can be advanced at a time.
void UpdateNextOutputTimestamp(Timestamp current_timestamp);
// Jitter-related variables.
std::unique_ptr<RandomBase> random_;
// Next packet to be emitted. Since packets may not align perfectly with
// next_output_timestamp_, the closest packet will be emitted.
Timestamp next_output_timestamp_;
// Lower bound for next timestamp.
//
// next_output_timestamp_ will be kept within the interval
// [next_output_timestamp_min_, next_output_timestamp_min_ + frame_time_usec_)
Timestamp next_output_timestamp_min_ = Timestamp::Unset();
// Indicates packet was emitted for current period (i.e. the period
// next_output_timestamp_ falls in.
bool packet_emitted_this_period_ = false;
};
// Strategy that applies Jitter without reflection based sampling.
//
// Used by PacketResamplerCalculator when Jitter is enabled and reflection is
// not enabled.
class JitterWithoutReflectionStrategy : public PacketResamplerStrategy {
public:
JitterWithoutReflectionStrategy(PacketResamplerCalculator* calculator)
: PacketResamplerStrategy(calculator) {}
absl::Status Open(CalculatorContext* cc) override;
absl::Status Close(CalculatorContext* cc) override;
absl::Status Process(CalculatorContext* cc) override;
private:
// Calculates the first sampled timestamp that incorporates a jittering
// offset.
void InitializeNextOutputTimestamp();
// Calculates the next sampled timestamp that incorporates a jittering offset.
void UpdateNextOutputTimestamp();
// Jitter-related variables.
std::unique_ptr<RandomBase> random_;
// Next packet to be emitted. Since packets may not align perfectly with
// next_output_timestamp_, the closest packet will be emitted.
Timestamp next_output_timestamp_;
// Lower bound for next timestamp.
//
// next_output_timestamp_ will be kept within the interval
// [next_output_timestamp_min_, next_output_timestamp_min_ + frame_time_usec_)
Timestamp next_output_timestamp_min_ = Timestamp::Unset();
// packet reservior used for sampling random packet out of partial period.
std::unique_ptr<PacketReservoir> packet_reservoir_;
// random number generator used in packet_reservior_.
std::unique_ptr<RandomBase> packet_reservoir_random_;
};
// Strategy that applies sampling without any jitter.
//
// Used by PacketResamplerCalculator when jitter is not enabled.
class NoJitterStrategy : public PacketResamplerStrategy {
public:
NoJitterStrategy(PacketResamplerCalculator* calculator)
: PacketResamplerStrategy(calculator) {}
absl::Status Open(CalculatorContext* cc) override;
absl::Status Close(CalculatorContext* cc) override;
absl::Status Process(CalculatorContext* cc) override;
private:
// Number of periods that have passed (= #packets sent to the output).
int64 period_count_;
// If specified, output timestamps are aligned with base_timestamp.
// Otherwise, they are aligned with the first input timestamp.
Timestamp base_timestamp_;
};
} // namespace mediapipe
#endif // MEDIAPIPE_CALCULATORS_CORE_PACKET_RESAMPLER_CALCULATOR_H_
mediapipe/calculators/core/packet_resampler_calculator.proto
+15
View File
@@ -68,8 +68,23 @@ message PacketResamplerCalculatorOptions {
// pseudo-random number generator does its job and the number of frames is
// sufficiently large, the average frame rate will be close to this value.
optional double jitter = 4;
// Enables reflection when applying jitter.
//
// This option is ignored when reproducible_sampling is true, in which case
// reflection will be used.
//
// New use cases should use reproducible_sampling = true, as
// jitter_with_reflection is deprecated and will be removed at some point.
optional bool jitter_with_reflection = 9 [default = false];
// If set, enabled reproducible sampling, allowing frames to be sampled
// without regards to where the stream starts. See
// packet_resampler_calculator.h for details.
//
// This enables reflection (ignoring jitter_with_reflection setting).
optional bool reproducible_sampling = 10 [default = false];
// If specified, output timestamps are aligned with base_timestamp.
// Otherwise, they are aligned with the first input timestamp.
//
mediapipe/calculators/core/packet_resampler_calculator_test.cc
+72
View File
@@ -30,6 +30,7 @@
namespace mediapipe {
using ::testing::ElementsAre;
namespace {
// A simple version of CalculatorRunner with built-in convenience
// methods for setting inputs from a vector and checking outputs
@@ -96,6 +97,77 @@ class SimpleRunner : public CalculatorRunner {
static int static_count_;
};
// Matcher for Packets with uint64 payload, comparing arg packet's
// timestamp and uint64 payload.
MATCHER_P2(PacketAtTimestamp, payload, timestamp,
absl::StrCat(negation ? "isn't" : "is", " a packet with payload ",
payload, " @ time ", timestamp)) {
if (timestamp != arg.Timestamp().Value()) {
*result_listener << "at incorrect timestamp = " << arg.Timestamp().Value();
return false;
}
int64 actual_payload = arg.template Get<int64>();
if (actual_payload != payload) {
*result_listener << "with incorrect payload = " << actual_payload;
return false;
}
return true;
}
// JitterWithReflectionStrategy child class which injects a specified stream
// of "random" numbers.
//
// Calculators are created through factory methods, making testing and injection
// tricky. This class utilizes a static variable, random_sequence, to pass
// the desired random sequence into the calculator.
class ReproducibleJitterWithReflectionStrategyForTesting
: public ReproducibleJitterWithReflectionStrategy {
public:
ReproducibleJitterWithReflectionStrategyForTesting(
PacketResamplerCalculator* calculator)
: ReproducibleJitterWithReflectionStrategy(calculator) {}
// Statically accessed random sequence to use for jitter with reflection.
//
// An EXPECT will fail if sequence is less than the number requested during
// processing.
static std::vector<uint64> random_sequence;
protected:
virtual uint64 GetNextRandom(uint64 n) {
EXPECT_LT(sequence_index_, random_sequence.size());
return random_sequence[sequence_index_++] % n;
}
private:
int32 sequence_index_ = 0;
};
std::vector<uint64>
ReproducibleJitterWithReflectionStrategyForTesting::random_sequence;
// PacketResamplerCalculator child class which injects a specified stream
// of "random" numbers.
//
// Calculators are created through factory methods, making testing and injection
// tricky. This class utilizes a static variable, random_sequence, to pass
// the desired random sequence into the calculator.
class ReproducibleResamplerCalculatorForTesting
: public PacketResamplerCalculator {
public:
static absl::Status GetContract(CalculatorContract* cc) {
return PacketResamplerCalculator::GetContract(cc);
}
protected:
std::unique_ptr<class PacketResamplerStrategy> GetSamplingStrategy(
const mediapipe::PacketResamplerCalculatorOptions& Options) {
return absl::make_unique<
ReproducibleJitterWithReflectionStrategyForTesting>(this);
}
};
REGISTER_CALCULATOR(ReproducibleResamplerCalculatorForTesting);
int SimpleRunner::static_count_ = 0;
TEST(PacketResamplerCalculatorTest, NoPacketsInStream) {
mediapipe/calculators/tensor/BUILD
+1 -1
View File
@@ -561,13 +561,13 @@ cc_test(
"//mediapipe/framework/formats:image_frame_opencv",
"//mediapipe/framework/formats:rect_cc_proto",
"//mediapipe/framework/formats:tensor",
"//mediapipe/framework/port:commandlineflags",
"//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:integral_types",
"//mediapipe/framework/port:opencv_core",
"//mediapipe/framework/port:opencv_imgcodecs",
"//mediapipe/framework/port:opencv_imgproc",
"//mediapipe/framework/port:parse_text_proto",
"@com_google_absl//absl/flags:flag",
"@com_google_absl//absl/memory",
"@com_google_absl//absl/strings",
],
mediapipe/calculators/tensor/image_to_tensor_calculator_test.cc
+1 -1
View File
@@ -15,6 +15,7 @@
#include <cmath>
#include <vector>
#include "absl/flags/flag.h"
#include "absl/memory/memory.h"
#include "absl/strings/substitute.h"
#include "mediapipe/calculators/tensor/image_to_tensor_converter.h"
@@ -28,7 +29,6 @@
#include "mediapipe/framework/formats/image_frame_opencv.h"
#include "mediapipe/framework/formats/rect.pb.h"
#include "mediapipe/framework/formats/tensor.h"
#include "mediapipe/framework/port/commandlineflags.h"
#include "mediapipe/framework/port/gtest.h"
#include "mediapipe/framework/port/integral_types.h"
#include "mediapipe/framework/port/opencv_core_inc.h"
mediapipe/calculators/tensor/inference_calculator.cc
+1 -1
View File
@@ -41,7 +41,7 @@ class InferenceCalculatorSelectorImpl
(options.has_delegate() && options.delegate().has_gpu());
if (should_use_gpu) {
impls.emplace_back("Metal");
impls.emplace_back("MlDrift");
impls.emplace_back("MlDriftWebGl");
impls.emplace_back("Gl");
}
impls.emplace_back("Cpu");
mediapipe/calculators/tensor/inference_calculator.h
+2 -2
View File
@@ -118,8 +118,8 @@ struct InferenceCalculatorGl : public InferenceCalculator {
static constexpr char kCalculatorName[] = "InferenceCalculatorGl";
};
struct InferenceCalculatorMlDrift : public InferenceCalculator {
static constexpr char kCalculatorName[] = "InferenceCalculatorMlDrift";
struct InferenceCalculatorMlDriftWebGl : public InferenceCalculator {
static constexpr char kCalculatorName[] = "InferenceCalculatorMlDriftWebGl";
};
struct InferenceCalculatorMetal : public InferenceCalculator {
mediapipe/calculators/tensor/inference_calculator.proto
+2 -2
View File
@@ -51,12 +51,12 @@ message InferenceCalculatorOptions {
// This option is valid for TFLite GPU delegate API2 only,
// Choose any of available APIs to force running inference using it.
enum API {
enum Api {
ANY = 0;
OPENGL = 1;
OPENCL = 2;
}
optional API api = 4 [default = ANY];
optional Api api = 4 [default = ANY];
// This option is valid for TFLite GPU delegate API2 only,
// Set to true to use 16-bit float precision. If max precision is needed,
mediapipe/calculators/tensor/inference_calculator_cpu.cc
+1 -1
View File
@@ -136,7 +136,7 @@ absl::Status InferenceCalculatorCpuImpl::LoadModel(CalculatorContext* cc) {
const auto& model = *model_packet_.Get();
tflite::ops::builtin::BuiltinOpResolver op_resolver =
kSideInCustomOpResolver(cc).GetOr(
tflite::ops::builtin::BuiltinOpResolver());
tflite::ops::builtin::BuiltinOpResolverWithoutDefaultDelegates());
tflite::InterpreterBuilder(model, op_resolver)(&interpreter_);
RET_CHECK(interpreter_);
mediapipe/calculators/tensor/inference_calculator_face_detection_test.cc
+1 -1
View File
@@ -59,7 +59,7 @@ const std::vector<Param>& GetParams() {
p.back().delegate.mutable_gpu();
#endif // TARGET_IPHONE_SIMULATOR
#if __EMSCRIPTEN__
p.push_back({"MlDrift", "MlDrift"});
p.push_back({"MlDriftWebGl", "MlDriftWebGl"});
p.back().delegate.mutable_gpu();
#endif // __EMSCRIPTEN__
#if __ANDROID__ && 0 // Disabled for now since emulator can't go GLESv3
mediapipe/calculators/tensor/inference_calculator_gl.cc
+3 -3
View File
@@ -63,7 +63,7 @@ class InferenceCalculatorGlImpl
mediapipe::GlCalculatorHelper gpu_helper_;
std::unique_ptr<tflite::gpu::TFLiteGPURunner> tflite_gpu_runner_;
bool allow_precision_loss_ = false;
mediapipe::InferenceCalculatorOptions::Delegate::Gpu::API
mediapipe::InferenceCalculatorOptions::Delegate::Gpu::Api
tflite_gpu_runner_api_;
#endif // MEDIAPIPE_TFLITE_GL_INFERENCE
@@ -244,7 +244,7 @@ absl::Status InferenceCalculatorGlImpl::InitTFLiteGPURunner(
const auto& model = *model_packet_.Get();
tflite::ops::builtin::BuiltinOpResolver op_resolver =
kSideInCustomOpResolver(cc).GetOr(
tflite::ops::builtin::BuiltinOpResolver());
tflite::ops::builtin::BuiltinOpResolverWithoutDefaultDelegates());
// Create runner
tflite::gpu::InferenceOptions options;
@@ -294,7 +294,7 @@ absl::Status InferenceCalculatorGlImpl::LoadModel(CalculatorContext* cc) {
const auto& model = *model_packet_.Get();
tflite::ops::builtin::BuiltinOpResolver op_resolver =
kSideInCustomOpResolver(cc).GetOr(
tflite::ops::builtin::BuiltinOpResolver());
tflite::ops::builtin::BuiltinOpResolverWithoutDefaultDelegates());
tflite::InterpreterBuilder(model, op_resolver)(&interpreter_);
RET_CHECK(interpreter_);
mediapipe/calculators/tensor/inference_calculator_metal.cc
+1 -1
View File
@@ -200,7 +200,7 @@ absl::Status InferenceCalculatorMetalImpl::LoadModel(CalculatorContext* cc) {
const auto& model = *model_packet_.Get();
tflite::ops::builtin::BuiltinOpResolver op_resolver =
kSideInCustomOpResolver(cc).GetOr(
tflite::ops::builtin::BuiltinOpResolver());
tflite::ops::builtin::BuiltinOpResolverWithoutDefaultDelegates());
tflite::InterpreterBuilder(model, op_resolver)(&interpreter_);
RET_CHECK(interpreter_);
mediapipe/calculators/tensorflow/BUILD
+5 -5
View File
@@ -892,13 +892,13 @@ cc_test(
"//mediapipe/framework:calculator_runner",
"//mediapipe/framework:packet",
"//mediapipe/framework/deps:file_path",
"//mediapipe/framework/port:commandlineflags",
"//mediapipe/framework/port:file_helpers",
"//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:parse_text_proto",
"//mediapipe/framework/port:status",
"//mediapipe/framework/tool:tag_map_helper",
"//mediapipe/framework/tool:validate_type",
"@com_google_absl//absl/flags:flag",
"@com_google_absl//absl/strings",
"@org_tensorflow//tensorflow/core:direct_session",
"@org_tensorflow//tensorflow/core:framework",
@@ -923,13 +923,13 @@ cc_test(
"//mediapipe/framework:packet",
"//mediapipe/framework:packet_generator_cc_proto",
"//mediapipe/framework/deps:file_path",
"//mediapipe/framework/port:commandlineflags",
"//mediapipe/framework/port:file_helpers",
"//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:parse_text_proto",
"//mediapipe/framework/port:status",
"//mediapipe/framework/tool:tag_map_helper",
"//mediapipe/framework/tool:validate_type",
"@com_google_absl//absl/flags:flag",
"@com_google_absl//absl/strings",
"@org_tensorflow//tensorflow/core:direct_session",
"@org_tensorflow//tensorflow/core:framework",
@@ -954,11 +954,11 @@ cc_test(
"//mediapipe/framework:packet",
"//mediapipe/framework:packet_generator_cc_proto",
"//mediapipe/framework/deps:file_path",
"//mediapipe/framework/port:commandlineflags",
"//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:parse_text_proto",
"//mediapipe/framework/tool:tag_map_helper",
"//mediapipe/framework/tool:validate_type",
"@com_google_absl//absl/flags:flag",
"@com_google_absl//absl/strings",
"@org_tensorflow//tensorflow/core:all_kernels",
"@org_tensorflow//tensorflow/core:direct_session",
@@ -981,11 +981,11 @@ cc_test(
"//mediapipe/framework:calculator_runner",
"//mediapipe/framework:packet",
"//mediapipe/framework/deps:file_path",
"//mediapipe/framework/port:commandlineflags",
"//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:parse_text_proto",
"//mediapipe/framework/tool:tag_map_helper",
"//mediapipe/framework/tool:validate_type",
"@com_google_absl//absl/flags:flag",
"@com_google_absl//absl/strings",
"@org_tensorflow//tensorflow/core:all_kernels",
"@org_tensorflow//tensorflow/core:direct_session",
@@ -1144,8 +1144,8 @@ cc_test(
":tensorflow_inference_calculator",
":tensorflow_session_from_frozen_graph_generator",
":tensorflow_session_from_frozen_graph_generator_cc_proto",
"@com_google_absl//absl/flags:flag",
"//mediapipe/framework/deps:file_path",
"//mediapipe/framework/port:commandlineflags",
"//mediapipe/framework/port:integral_types",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:calculator_runner",
mediapipe/calculators/tensorflow/tensorflow_inference_calculator_test.cc
+1 -1
View File
@@ -16,12 +16,12 @@
#include <string>
#include <vector>
#include "absl/flags/flag.h"
#include "mediapipe/calculators/tensorflow/tensorflow_inference_calculator.pb.h"
#include "mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_generator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_runner.h"
#include "mediapipe/framework/deps/file_path.h"
#include "mediapipe/framework/port/commandlineflags.h"
#include "mediapipe/framework/port/gmock.h"
#include "mediapipe/framework/port/gtest.h"
#include "mediapipe/framework/port/integral_types.h"
mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_calculator_test.cc
+1 -1
View File
@@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include "absl/flags/flag.h"
#include "absl/strings/substitute.h"
#include "mediapipe/calculators/tensorflow/tensorflow_session.h"
#include "mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_calculator.pb.h"
@@ -19,7 +20,6 @@
#include "mediapipe/framework/calculator_runner.h"
#include "mediapipe/framework/deps/file_path.h"
#include "mediapipe/framework/packet.h"
#include "mediapipe/framework/port/commandlineflags.h"
#include "mediapipe/framework/port/file_helpers.h"
#include "mediapipe/framework/port/gmock.h"
#include "mediapipe/framework/port/gtest.h"
mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_generator_test.cc
+1 -1
View File
@@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include "absl/flags/flag.h"
#include "absl/strings/substitute.h"
#include "mediapipe/calculators/tensorflow/tensorflow_session.h"
#include "mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_generator.pb.h"
@@ -19,7 +20,6 @@
#include "mediapipe/framework/deps/file_path.h"
#include "mediapipe/framework/packet.h"
#include "mediapipe/framework/packet_generator.pb.h"
#include "mediapipe/framework/port/commandlineflags.h"
#include "mediapipe/framework/port/file_helpers.h"
#include "mediapipe/framework/port/gmock.h"
#include "mediapipe/framework/port/gtest.h"
mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator_test.cc
+1 -1
View File
@@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include "absl/flags/flag.h"
#include "absl/strings/str_replace.h"
#include "mediapipe/calculators/tensorflow/tensorflow_session.h"
#include "mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator.pb.h"
@@ -20,7 +21,6 @@
#include "mediapipe/framework/calculator_runner.h"
#include "mediapipe/framework/deps/file_path.h"
#include "mediapipe/framework/packet.h"
#include "mediapipe/framework/port/commandlineflags.h"
#include "mediapipe/framework/port/gmock.h"
#include "mediapipe/framework/port/gtest.h"
#include "mediapipe/framework/port/parse_text_proto.h"
mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator_test.cc
+1 -1
View File
@@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include "absl/flags/flag.h"
#include "absl/strings/str_replace.h"
#include "mediapipe/calculators/tensorflow/tensorflow_session.h"
#include "mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator.pb.h"
@@ -19,7 +20,6 @@
#include "mediapipe/framework/deps/file_path.h"
#include "mediapipe/framework/packet.h"
#include "mediapipe/framework/packet_generator.pb.h"
#include "mediapipe/framework/port/commandlineflags.h"
#include "mediapipe/framework/port/gmock.h"
#include "mediapipe/framework/port/gtest.h"
#include "mediapipe/framework/port/parse_text_proto.h"
mediapipe/calculators/tensorflow/unpack_media_sequence_calculator.cc
+1
View File
@@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include "absl/container/flat_hash_map.h"
#include "absl/strings/match.h"
#include "mediapipe/calculators/core/packet_resampler_calculator.pb.h"
#include "mediapipe/calculators/tensorflow/unpack_media_sequence_calculator.pb.h"
mediapipe/calculators/tensorflow/unpack_media_sequence_calculator.proto
-7
View File
@@ -56,11 +56,4 @@ message UnpackMediaSequenceCalculatorOptions {
// the clip start and end times and outputs these for the
// AudioDecoderCalculator to consume.
optional AudioDecoderOptions base_audio_decoder_options = 9;
optional string keypoint_names = 10 [
default =
"NOSE,LEFT_EAR,RIGHT_EAR,LEFT_SHOULDER,RIGHT_SHOULDER,LEFT_FORE_PAW,RIGHT_FORE_PAW,LEFT_HIP,RIGHT_HIP,LEFT_HIND_PAW,RIGHT_HIND_PAW,ROOT_TAIL"
];
// When the keypoint doesn't exists, output this default value.
optional float default_keypoint_location = 11 [default = -1.0];
}
mediapipe/calculators/tflite/BUILD
+1 -1
View File
@@ -147,11 +147,11 @@ cc_test(
"//mediapipe/framework:calculator_runner",
"//mediapipe/framework/deps:file_path",
"//mediapipe/framework/formats/object_detection:anchor_cc_proto",
"//mediapipe/framework/port:commandlineflags",
"//mediapipe/framework/port:file_helpers",
"//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:integral_types",
"//mediapipe/framework/port:parse_text_proto",
"@com_google_absl//absl/flags:flag",
],
)
mediapipe/calculators/tflite/ssd_anchors_calculator_test.cc
+1 -1
View File
@@ -12,11 +12,11 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include "absl/flags/flag.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_runner.h"
#include "mediapipe/framework/deps/file_path.h"
#include "mediapipe/framework/formats/object_detection/anchor.pb.h"
#include "mediapipe/framework/port/commandlineflags.h"
#include "mediapipe/framework/port/file_helpers.h"
#include "mediapipe/framework/port/gmock.h"
#include "mediapipe/framework/port/gtest.h"
mediapipe/calculators/tflite/tflite_inference_calculator.cc
+23 -12
View File
@@ -278,7 +278,7 @@ class TfLiteInferenceCalculator : public CalculatorBase {
bool use_advanced_gpu_api_ = false;
bool allow_precision_loss_ = false;
mediapipe::TfLiteInferenceCalculatorOptions::Delegate::Gpu::API
mediapipe::TfLiteInferenceCalculatorOptions::Delegate::Gpu::Api
tflite_gpu_runner_api_;
bool use_kernel_caching_ = false;
@@ -702,11 +702,16 @@ absl::Status TfLiteInferenceCalculator::InitTFLiteGPURunner(
#if MEDIAPIPE_TFLITE_GL_INFERENCE
ASSIGN_OR_RETURN(model_packet_, GetModelAsPacket(*cc));
const auto& model = *model_packet_.Get<TfLiteModelPtr>();
tflite::ops::builtin::BuiltinOpResolver op_resolver;
tflite::ops::builtin::BuiltinOpResolverWithoutDefaultDelegates
default_op_resolver;
auto op_resolver_ptr =
static_cast<const tflite::ops::builtin::BuiltinOpResolver*>(
&default_op_resolver);
if (cc->InputSidePackets().HasTag("CUSTOM_OP_RESOLVER")) {
op_resolver = cc->InputSidePackets()
.Tag("CUSTOM_OP_RESOLVER")
.Get<tflite::ops::builtin::BuiltinOpResolver>();
op_resolver_ptr = &(cc->InputSidePackets()
.Tag("CUSTOM_OP_RESOLVER")
.Get<tflite::ops::builtin::BuiltinOpResolver>());
}
// Create runner
@@ -733,7 +738,7 @@ absl::Status TfLiteInferenceCalculator::InitTFLiteGPURunner(
}
}
MP_RETURN_IF_ERROR(
tflite_gpu_runner_->InitializeWithModel(model, op_resolver));
tflite_gpu_runner_->InitializeWithModel(model, *op_resolver_ptr));
// Allocate interpreter memory for cpu output.
if (!gpu_output_) {
@@ -786,18 +791,24 @@ absl::Status TfLiteInferenceCalculator::LoadModel(CalculatorContext* cc) {
ASSIGN_OR_RETURN(model_packet_, GetModelAsPacket(*cc));
const auto& model = *model_packet_.Get<TfLiteModelPtr>();
tflite::ops::builtin::BuiltinOpResolver op_resolver;
tflite::ops::builtin::BuiltinOpResolverWithoutDefaultDelegates
default_op_resolver;
auto op_resolver_ptr =
static_cast<const tflite::ops::builtin::BuiltinOpResolver*>(
&default_op_resolver);
if (cc->InputSidePackets().HasTag("CUSTOM_OP_RESOLVER")) {
op_resolver = cc->InputSidePackets()
.Tag("CUSTOM_OP_RESOLVER")
.Get<tflite::ops::builtin::BuiltinOpResolver>();
op_resolver_ptr = &(cc->InputSidePackets()
.Tag("CUSTOM_OP_RESOLVER")
.Get<tflite::ops::builtin::BuiltinOpResolver>());
}
#if defined(MEDIAPIPE_EDGE_TPU)
interpreter_ =
BuildEdgeTpuInterpreter(model, &op_resolver, edgetpu_context_.get());
BuildEdgeTpuInterpreter(model, op_resolver_ptr, edgetpu_context_.get());
#else
tflite::InterpreterBuilder(model, op_resolver)(&interpreter_);
tflite::InterpreterBuilder(model, *op_resolver_ptr)(&interpreter_);
#endif // MEDIAPIPE_EDGE_TPU
RET_CHECK(interpreter_);
mediapipe/calculators/tflite/tflite_inference_calculator.proto
+2 -2
View File
@@ -51,12 +51,12 @@ message TfLiteInferenceCalculatorOptions {
// This option is valid for TFLite GPU delegate API2 only,
// Choose any of available APIs to force running inference using it.
enum API {
enum Api {
ANY = 0;
OPENGL = 1;
OPENCL = 2;
}
optional API api = 4 [default = ANY];
optional Api api = 4 [default = ANY];
// This option is valid for TFLite GPU delegate API2 only,
// Set to true to use 16-bit float precision. If max precision is needed,
mediapipe/calculators/util/BUILD
+59 -1
View File
@@ -841,12 +841,39 @@ cc_library(
"//mediapipe/framework:timestamp",
"//mediapipe/framework/formats:landmark_cc_proto",
"//mediapipe/framework/port:ret_check",
"//mediapipe/util/filtering:one_euro_filter",
"//mediapipe/util/filtering:relative_velocity_filter",
"@com_google_absl//absl/algorithm:container",
],
alwayslink = 1,
)
mediapipe_proto_library(
name = "visibility_smoothing_calculator_proto",
srcs = ["visibility_smoothing_calculator.proto"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
cc_library(
name = "visibility_smoothing_calculator",
srcs = ["visibility_smoothing_calculator.cc"],
visibility = ["//visibility:public"],
deps = [
":visibility_smoothing_calculator_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:timestamp",
"//mediapipe/framework/formats:landmark_cc_proto",
"//mediapipe/framework/port:ret_check",
"//mediapipe/util/filtering:low_pass_filter",
"@com_google_absl//absl/algorithm:container",
],
alwayslink = 1,
)
cc_library(
name = "landmarks_to_floats_calculator",
srcs = ["landmarks_to_floats_calculator.cc"],
@@ -858,7 +885,7 @@ cc_library(
"//mediapipe/framework/formats:matrix",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"@eigen_archive//:eigen",
"@eigen_archive//:eigen3",
],
alwayslink = 1,
)
@@ -1194,3 +1221,34 @@ cc_library(
}),
alwayslink = 1,
)
cc_library(
name = "detection_classifications_merger_calculator",
srcs = ["detection_classifications_merger_calculator.cc"],
deps = [
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/api2:node",
"//mediapipe/framework/formats:classification_cc_proto",
"//mediapipe/framework/formats:detection_cc_proto",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"//mediapipe/framework/port:statusor",
"@com_google_absl//absl/strings",
],
alwayslink = 1,
)
cc_test(
name = "detection_classifications_merger_calculator_test",
srcs = ["detection_classifications_merger_calculator_test.cc"],
deps = [
":detection_classifications_merger_calculator",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:calculator_runner",
"//mediapipe/framework/deps:message_matchers",
"//mediapipe/framework/formats:classification_cc_proto",
"//mediapipe/framework/formats:detection_cc_proto",
"//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:parse_text_proto",
],
)
mediapipe/calculators/util/detection_classifications_merger_calculator.cc
+149
View File
@@ -0,0 +1,149 @@
// Copyright 2021 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "absl/strings/substitute.h"
#include "mediapipe/framework/api2/node.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/classification.pb.h"
#include "mediapipe/framework/formats/detection.pb.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/framework/port/statusor.h"
namespace mediapipe {
namespace api2 {
namespace {} // namespace
// Replaces the classification labels and scores from the input `Detection` with
// the ones provided into the input `ClassificationList`. Namely:
// * `label_id[i]` becomes `classification[i].index`
// * `score[i]` becomes `classification[i].score`
// * `label[i]` becomes `classification[i].label` (if present)
//
// In case the input `ClassificationList` contains no results (i.e.
// `classification` is empty, which may happen if the classifier uses a score
// threshold and no confident enough result were returned), the input
// `Detection` is returned unchanged.
//
// This is specifically designed for two-stage detection cascades where the
// detections returned by a standalone detector (typically a class-agnostic
// localizer) are fed e.g. into a `TfLiteTaskImageClassifierCalculator` through
// the optional "RECT" or "NORM_RECT" input, e.g:
//
// node {
// calculator: "DetectionsToRectsCalculator"
// # Output of an upstream object detector.
// input_stream: "DETECTION:detection"
// output_stream: "NORM_RECT:norm_rect"
// }
// node {
// calculator: "TfLiteTaskImageClassifierCalculator"
// input_stream: "IMAGE:image"
// input_stream: "NORM_RECT:norm_rect"
// output_stream: "CLASSIFICATION_RESULT:classification_result"
// }
// node {
// calculator: "TfLiteTaskClassificationResultToClassificationsCalculator"
// input_stream: "CLASSIFICATION_RESULT:classification_result"
// output_stream: "CLASSIFICATION_LIST:classification_list"
// }
// node {
// calculator: "DetectionClassificationsMergerCalculator"
// input_stream: "INPUT_DETECTION:detection"
// input_stream: "CLASSIFICATION_LIST:classification_list"
// # Final output.
// output_stream: "OUTPUT_DETECTION:classified_detection"
// }
//
// Inputs:
// INPUT_DETECTION: `Detection` proto.
// CLASSIFICATION_LIST: `ClassificationList` proto.
//
// Output:
// OUTPUT_DETECTION: modified `Detection` proto.
class DetectionClassificationsMergerCalculator : public Node {
public:
static constexpr Input<Detection> kInputDetection{"INPUT_DETECTION"};
static constexpr Input<ClassificationList> kClassificationList{
"CLASSIFICATION_LIST"};
static constexpr Output<Detection> kOutputDetection{"OUTPUT_DETECTION"};
MEDIAPIPE_NODE_CONTRACT(kInputDetection, kClassificationList,
kOutputDetection);
absl::Status Process(CalculatorContext* cc) override;
};
MEDIAPIPE_REGISTER_NODE(DetectionClassificationsMergerCalculator);
absl::Status DetectionClassificationsMergerCalculator::Process(
CalculatorContext* cc) {
if (kInputDetection(cc).IsEmpty() && kClassificationList(cc).IsEmpty()) {
return absl::OkStatus();
}
RET_CHECK(!kInputDetection(cc).IsEmpty());
RET_CHECK(!kClassificationList(cc).IsEmpty());
Detection detection = *kInputDetection(cc);
const ClassificationList& classification_list = *kClassificationList(cc);
// Update input detection only if classification did return results.
if (classification_list.classification_size() != 0) {
detection.clear_label_id();
detection.clear_score();
detection.clear_label();
detection.clear_display_name();
for (const auto& classification : classification_list.classification()) {
if (!classification.has_index()) {
return absl::InvalidArgumentError(
"Missing required 'index' field in Classification proto.");
}
detection.add_label_id(classification.index());
if (!classification.has_score()) {
return absl::InvalidArgumentError(
"Missing required 'score' field in Classification proto.");
}
detection.add_score(classification.score());
if (classification.has_label()) {
detection.add_label(classification.label());
}
if (classification.has_display_name()) {
detection.add_display_name(classification.display_name());
}
}
// Post-conversion sanity checks.
if (detection.label_size() != 0 &&
detection.label_size() != detection.label_id_size()) {
return absl::InvalidArgumentError(absl::Substitute(
"Each input Classification is expected to either always or never "
"provide a 'label' field. Found $0 'label' fields for $1 "
"'Classification' objects.",
/*$0=*/detection.label_size(), /*$1=*/detection.label_id_size()));
}
if (detection.display_name_size() != 0 &&
detection.display_name_size() != detection.label_id_size()) {
return absl::InvalidArgumentError(absl::Substitute(
"Each input Classification is expected to either always or never "
"provide a 'display_name' field. Found $0 'display_name' fields for "
"$1 'Classification' objects.",
/*$0=*/detection.display_name_size(),
/*$1=*/detection.label_id_size()));
}
}
kOutputDetection(cc).Send(detection);
return absl::OkStatus();
}
} // namespace api2
} // namespace mediapipe
mediapipe/calculators/util/detection_classifications_merger_calculator_test.cc
+320
View File
@@ -0,0 +1,320 @@
// Copyright 2021 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_runner.h"
#include "mediapipe/framework/deps/message_matchers.h"
#include "mediapipe/framework/formats/classification.pb.h"
#include "mediapipe/framework/formats/detection.pb.h"
#include "mediapipe/framework/port/gtest.h"
#include "mediapipe/framework/port/parse_text_proto.h"
#include "mediapipe/framework/port/status_matchers.h"
namespace mediapipe {
namespace {
constexpr char kGraphConfig[] = R"(
input_stream: "input_detection"
input_stream: "classification_list"
output_stream: "output_detection"
node {
calculator: "DetectionClassificationsMergerCalculator"
input_stream: "INPUT_DETECTION:input_detection"
input_stream: "CLASSIFICATION_LIST:classification_list"
output_stream: "OUTPUT_DETECTION:output_detection"
}
)";
constexpr char kInputDetection[] = R"(
label: "entity"
label_id: 1
score: 0.9
location_data {
format: BOUNDING_BOX
bounding_box { xmin: 50 ymin: 60 width: 70 height: 80 }
}
display_name: "Entity"
)";
// Checks that the input Detection is returned unchanged if the input
// ClassificationList does not contain any result.
TEST(DetectionClassificationsMergerCalculator, SucceedsWithNoClassification) {
auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(kGraphConfig);
// Prepare input packets.
const Detection& input_detection =
ParseTextProtoOrDie<Detection>(kInputDetection);
Packet input_detection_packet =
MakePacket<Detection>(input_detection).At(Timestamp(0));
const ClassificationList& classification_list =
ParseTextProtoOrDie<ClassificationList>("");
Packet classification_list_packet =
MakePacket<ClassificationList>(classification_list).At(Timestamp(0));
// Catch output.
std::vector<Packet> output_packets;
tool::AddVectorSink("output_detection", &graph_config, &output_packets);
// Run the graph.
CalculatorGraph graph;
MP_ASSERT_OK(graph.Initialize(graph_config));
MP_ASSERT_OK(graph.StartRun({}));
MP_ASSERT_OK(
graph.AddPacketToInputStream("input_detection", input_detection_packet));
MP_ASSERT_OK(graph.AddPacketToInputStream("classification_list",
classification_list_packet));
MP_ASSERT_OK(graph.WaitUntilIdle());
// Get and validate output.
EXPECT_THAT(output_packets, testing::SizeIs(1));
const Detection& output_detection = output_packets[0].Get<Detection>();
EXPECT_THAT(output_detection, mediapipe::EqualsProto(input_detection));
}
// Checks that merging succeeds when the input ClassificationList includes
// labels and display names.
TEST(DetectionClassificationsMergerCalculator,
SucceedsWithLabelsAndDisplayNames) {
auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(kGraphConfig);
// Prepare input packets.
const Detection& input_detection =
ParseTextProtoOrDie<Detection>(kInputDetection);
Packet input_detection_packet =
MakePacket<Detection>(input_detection).At(Timestamp(0));
const ClassificationList& classification_list =
ParseTextProtoOrDie<ClassificationList>(R"(
classification { index: 11 score: 0.5 label: "dog" display_name: "Dog" }
classification { index: 12 score: 0.4 label: "fox" display_name: "Fox" }
)");
Packet classification_list_packet =
MakePacket<ClassificationList>(classification_list).At(Timestamp(0));
// Catch output.
std::vector<Packet> output_packets;
tool::AddVectorSink("output_detection", &graph_config, &output_packets);
// Run the graph.
CalculatorGraph graph;
MP_ASSERT_OK(graph.Initialize(graph_config));
MP_ASSERT_OK(graph.StartRun({}));
MP_ASSERT_OK(
graph.AddPacketToInputStream("input_detection", input_detection_packet));
MP_ASSERT_OK(graph.AddPacketToInputStream("classification_list",
classification_list_packet));
MP_ASSERT_OK(graph.WaitUntilIdle());
// Get and validate output.
EXPECT_THAT(output_packets, testing::SizeIs(1));
const Detection& output_detection = output_packets[0].Get<Detection>();
EXPECT_THAT(output_detection,
mediapipe::EqualsProto(ParseTextProtoOrDie<Detection>(R"(
label: "dog"
label: "fox"
label_id: 11
label_id: 12
score: 0.5
score: 0.4
location_data {
format: BOUNDING_BOX
bounding_box { xmin: 50 ymin: 60 width: 70 height: 80 }
}
display_name: "Dog"
display_name: "Fox"
)")));
}
// Checks that merging succeeds when the input ClassificationList doesn't
// include labels and display names.
TEST(DetectionClassificationsMergerCalculator,
SucceedsWithoutLabelsAndDisplayNames) {
auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(kGraphConfig);
// Prepare input packets.
const Detection& input_detection =
ParseTextProtoOrDie<Detection>(kInputDetection);
Packet input_detection_packet =
MakePacket<Detection>(input_detection).At(Timestamp(0));
const ClassificationList& classification_list =
ParseTextProtoOrDie<ClassificationList>(R"(
classification { index: 11 score: 0.5 }
classification { index: 12 score: 0.4 }
)");
Packet classification_list_packet =
MakePacket<ClassificationList>(classification_list).At(Timestamp(0));
// Catch output.
std::vector<Packet> output_packets;
tool::AddVectorSink("output_detection", &graph_config, &output_packets);
// Run the graph.
CalculatorGraph graph;
MP_ASSERT_OK(graph.Initialize(graph_config));
MP_ASSERT_OK(graph.StartRun({}));
MP_ASSERT_OK(
graph.AddPacketToInputStream("input_detection", input_detection_packet));
MP_ASSERT_OK(graph.AddPacketToInputStream("classification_list",
classification_list_packet));
MP_ASSERT_OK(graph.WaitUntilIdle());
// Get and validate output.
EXPECT_THAT(output_packets, testing::SizeIs(1));
const Detection& output_detection = output_packets[0].Get<Detection>();
EXPECT_THAT(output_detection,
mediapipe::EqualsProto(ParseTextProtoOrDie<Detection>(R"(
label_id: 11
label_id: 12
score: 0.5
score: 0.4
location_data {
format: BOUNDING_BOX
bounding_box { xmin: 50 ymin: 60 width: 70 height: 80 }
}
)")));
}
// Checks that merging fails if the input ClassificationList misses mandatory
// "index" field.
TEST(DetectionClassificationsMergerCalculator, FailsWithMissingIndex) {
auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(kGraphConfig);
// Prepare input packets.
const Detection& input_detection =
ParseTextProtoOrDie<Detection>(kInputDetection);
Packet input_detection_packet =
MakePacket<Detection>(input_detection).At(Timestamp(0));
const ClassificationList& classification_list =
ParseTextProtoOrDie<ClassificationList>(R"(
classification { score: 0.5 label: "dog" }
)");
Packet classification_list_packet =
MakePacket<ClassificationList>(classification_list).At(Timestamp(0));
// Catch output.
std::vector<Packet> output_packets;
tool::AddVectorSink("output_detection", &graph_config, &output_packets);
// Run the graph.
CalculatorGraph graph;
MP_ASSERT_OK(graph.Initialize(graph_config));
MP_ASSERT_OK(graph.StartRun({}));
MP_ASSERT_OK(
graph.AddPacketToInputStream("input_detection", input_detection_packet));
MP_ASSERT_OK(graph.AddPacketToInputStream("classification_list",
classification_list_packet));
ASSERT_EQ(graph.WaitUntilIdle().code(), absl::StatusCode::kInvalidArgument);
}
// Checks that merging fails if the input ClassificationList misses mandatory
// "score" field.
TEST(DetectionClassificationsMergerCalculator, FailsWithMissingScore) {
auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(kGraphConfig);
// Prepare input packets.
const Detection& input_detection =
ParseTextProtoOrDie<Detection>(kInputDetection);
Packet input_detection_packet =
MakePacket<Detection>(input_detection).At(Timestamp(0));
const ClassificationList& classification_list =
ParseTextProtoOrDie<ClassificationList>(R"(
classification { index: 11 label: "dog" }
)");
Packet classification_list_packet =
MakePacket<ClassificationList>(classification_list).At(Timestamp(0));
// Catch output.
std::vector<Packet> output_packets;
tool::AddVectorSink("output_detection", &graph_config, &output_packets);
// Run the graph.
CalculatorGraph graph;
MP_ASSERT_OK(graph.Initialize(graph_config));
MP_ASSERT_OK(graph.StartRun({}));
MP_ASSERT_OK(
graph.AddPacketToInputStream("input_detection", input_detection_packet));
MP_ASSERT_OK(graph.AddPacketToInputStream("classification_list",
classification_list_packet));
ASSERT_EQ(graph.WaitUntilIdle().code(), absl::StatusCode::kInvalidArgument);
}
// Checks that merging fails if the input ClassificationList has an
// inconsistent number of labels.
TEST(DetectionClassificationsMergerCalculator,
FailsWithInconsistentNumberOfLabels) {
auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(kGraphConfig);
// Prepare input packets.
const Detection& input_detection =
ParseTextProtoOrDie<Detection>(kInputDetection);
Packet input_detection_packet =
MakePacket<Detection>(input_detection).At(Timestamp(0));
const ClassificationList& classification_list =
ParseTextProtoOrDie<ClassificationList>(R"(
classification { index: 11 score: 0.5 label: "dog" display_name: "Dog" }
classification { index: 12 score: 0.4 display_name: "Fox" }
)");
Packet classification_list_packet =
MakePacket<ClassificationList>(classification_list).At(Timestamp(0));
// Catch output.
std::vector<Packet> output_packets;
tool::AddVectorSink("output_detection", &graph_config, &output_packets);
// Run the graph.
CalculatorGraph graph;
MP_ASSERT_OK(graph.Initialize(graph_config));
MP_ASSERT_OK(graph.StartRun({}));
MP_ASSERT_OK(
graph.AddPacketToInputStream("input_detection", input_detection_packet));
MP_ASSERT_OK(graph.AddPacketToInputStream("classification_list",
classification_list_packet));
ASSERT_EQ(graph.WaitUntilIdle().code(), absl::StatusCode::kInvalidArgument);
}
// Checks that merging fails if the input ClassificationList has an
// inconsistent number of display names.
TEST(DetectionClassificationsMergerCalculator,
FailsWithInconsistentNumberOfDisplayNames) {
auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(kGraphConfig);
// Prepare input packets.
const Detection& input_detection =
ParseTextProtoOrDie<Detection>(kInputDetection);
Packet input_detection_packet =
MakePacket<Detection>(input_detection).At(Timestamp(0));
const ClassificationList& classification_list =
ParseTextProtoOrDie<ClassificationList>(R"(
classification { index: 11 score: 0.5 label: "dog" }
classification { index: 12 score: 0.4 label: "fox" display_name: "Fox" }
)");
Packet classification_list_packet =
MakePacket<ClassificationList>(classification_list).At(Timestamp(0));
// Catch output.
std::vector<Packet> output_packets;
tool::AddVectorSink("output_detection", &graph_config, &output_packets);
// Run the graph.
CalculatorGraph graph;
MP_ASSERT_OK(graph.Initialize(graph_config));
MP_ASSERT_OK(graph.StartRun({}));
MP_ASSERT_OK(
graph.AddPacketToInputStream("input_detection", input_detection_packet));
MP_ASSERT_OK(graph.AddPacketToInputStream("classification_list",
classification_list_packet));
ASSERT_EQ(graph.WaitUntilIdle().code(), absl::StatusCode::kInvalidArgument);
}
} // namespace
} // namespace mediapipe
mediapipe/calculators/util/landmarks_smoothing_calculator.cc
+198 -58
View File
@@ -12,12 +12,15 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include <memory>
#include "absl/algorithm/container.h"
#include "mediapipe/calculators/util/landmarks_smoothing_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/landmark.pb.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/timestamp.h"
#include "mediapipe/util/filtering/one_euro_filter.h"
#include "mediapipe/util/filtering/relative_velocity_filter.h"
namespace mediapipe {
@@ -25,19 +28,54 @@ namespace mediapipe {
namespace {
constexpr char kNormalizedLandmarksTag[] = "NORM_LANDMARKS";
constexpr char kLandmarksTag[] = "LANDMARKS";
constexpr char kImageSizeTag[] = "IMAGE_SIZE";
constexpr char kNormalizedFilteredLandmarksTag[] = "NORM_FILTERED_LANDMARKS";
constexpr char kFilteredLandmarksTag[] = "FILTERED_LANDMARKS";
using mediapipe::OneEuroFilter;
using mediapipe::RelativeVelocityFilter;
void NormalizedLandmarksToLandmarks(
const NormalizedLandmarkList& norm_landmarks, const int image_width,
const int image_height, LandmarkList* landmarks) {
for (int i = 0; i < norm_landmarks.landmark_size(); ++i) {
const auto& norm_landmark = norm_landmarks.landmark(i);
auto* landmark = landmarks->add_landmark();
landmark->set_x(norm_landmark.x() * image_width);
landmark->set_y(norm_landmark.y() * image_height);
// Scale Z the same way as X (using image width).
landmark->set_z(norm_landmark.z() * image_width);
landmark->set_visibility(norm_landmark.visibility());
landmark->set_presence(norm_landmark.presence());
}
}
void LandmarksToNormalizedLandmarks(const LandmarkList& landmarks,
const int image_width,
const int image_height,
NormalizedLandmarkList* norm_landmarks) {
for (int i = 0; i < landmarks.landmark_size(); ++i) {
const auto& landmark = landmarks.landmark(i);
auto* norm_landmark = norm_landmarks->add_landmark();
norm_landmark->set_x(landmark.x() / image_width);
norm_landmark->set_y(landmark.y() / image_height);
// Scale Z the same way as X (using image width).
norm_landmark->set_z(landmark.z() / image_width);
norm_landmark->set_visibility(landmark.visibility());
norm_landmark->set_presence(landmark.presence());
}
}
// Estimate object scale to use its inverse value as velocity scale for
// RelativeVelocityFilter. If value will be too small (less than
// `options_.min_allowed_object_scale`) smoothing will be disabled and
// landmarks will be returned as is.
// Object scale is calculated as average between bounding box width and height
// with sides parallel to axis.
float GetObjectScale(const NormalizedLandmarkList& landmarks, int image_width,
int image_height) {
float GetObjectScale(const LandmarkList& landmarks) {
const auto& lm_minmax_x = absl::c_minmax_element(
landmarks.landmark(),
[](const auto& a, const auto& b) { return a.x() < b.x(); });
@@ -50,8 +88,8 @@ float GetObjectScale(const NormalizedLandmarkList& landmarks, int image_width,
const float y_min = lm_minmax_y.first->y();
const float y_max = lm_minmax_y.second->y();
const float object_width = (x_max - x_min) * image_width;
const float object_height = (y_max - y_min) * image_height;
const float object_width = x_max - x_min;
const float object_height = y_max - y_min;
return (object_width + object_height) / 2.0f;
}
@@ -63,19 +101,17 @@ class LandmarksFilter {
virtual absl::Status Reset() { return absl::OkStatus(); }
virtual absl::Status Apply(const NormalizedLandmarkList& in_landmarks,
const std::pair<int, int>& image_size,
virtual absl::Status Apply(const LandmarkList& in_landmarks,
const absl::Duration& timestamp,
NormalizedLandmarkList* out_landmarks) = 0;
LandmarkList* out_landmarks) = 0;
};
// Returns landmarks as is without smoothing.
class NoFilter : public LandmarksFilter {
public:
absl::Status Apply(const NormalizedLandmarkList& in_landmarks,
const std::pair<int, int>& image_size,
absl::Status Apply(const LandmarkList& in_landmarks,
const absl::Duration& timestamp,
NormalizedLandmarkList* out_landmarks) override {
LandmarkList* out_landmarks) override {
*out_landmarks = in_landmarks;
return absl::OkStatus();
}
@@ -85,10 +121,11 @@ class NoFilter : public LandmarksFilter {
class VelocityFilter : public LandmarksFilter {
public:
VelocityFilter(int window_size, float velocity_scale,
float min_allowed_object_scale)
float min_allowed_object_scale, bool disable_value_scaling)
: window_size_(window_size),
velocity_scale_(velocity_scale),
min_allowed_object_scale_(min_allowed_object_scale) {}
min_allowed_object_scale_(min_allowed_object_scale),
disable_value_scaling_(disable_value_scaling) {}
absl::Status Reset() override {
x_filters_.clear();
@@ -97,45 +134,37 @@ class VelocityFilter : public LandmarksFilter {
return absl::OkStatus();
}
absl::Status Apply(const NormalizedLandmarkList& in_landmarks,
const std::pair<int, int>& image_size,
absl::Status Apply(const LandmarkList& in_landmarks,
const absl::Duration& timestamp,
NormalizedLandmarkList* out_landmarks) override {
// Get image size.
int image_width;
int image_height;
std::tie(image_width, image_height) = image_size;
LandmarkList* out_landmarks) override {
// Get value scale as inverse value of the object scale.
// If value is too small smoothing will be disabled and landmarks will be
// returned as is.
const float object_scale =
GetObjectScale(in_landmarks, image_width, image_height);
if (object_scale < min_allowed_object_scale_) {
*out_landmarks = in_landmarks;
return absl::OkStatus();
float value_scale = 1.0f;
if (!disable_value_scaling_) {
const float object_scale = GetObjectScale(in_landmarks);
if (object_scale < min_allowed_object_scale_) {
*out_landmarks = in_landmarks;
return absl::OkStatus();
}
value_scale = 1.0f / object_scale;
}
const float value_scale = 1.0f / object_scale;
// Initialize filters once.
MP_RETURN_IF_ERROR(InitializeFiltersIfEmpty(in_landmarks.landmark_size()));
// Filter landmarks. Every axis of every landmark is filtered separately.
for (int i = 0; i < in_landmarks.landmark_size(); ++i) {
const NormalizedLandmark& in_landmark = in_landmarks.landmark(i);
const auto& in_landmark = in_landmarks.landmark(i);
NormalizedLandmark* out_landmark = out_landmarks->add_landmark();
auto* out_landmark = out_landmarks->add_landmark();
*out_landmark = in_landmark;
out_landmark->set_x(x_filters_[i].Apply(timestamp, value_scale,
in_landmark.x() * image_width) /
image_width);
out_landmark->set_y(y_filters_[i].Apply(timestamp, value_scale,
in_landmark.y() * image_height) /
image_height);
// Scale Z the save was as X (using image width).
out_landmark->set_z(z_filters_[i].Apply(timestamp, value_scale,
in_landmark.z() * image_width) /
image_width);
out_landmark->set_x(
x_filters_[i].Apply(timestamp, value_scale, in_landmark.x()));
out_landmark->set_y(
y_filters_[i].Apply(timestamp, value_scale, in_landmark.y()));
out_landmark->set_z(
z_filters_[i].Apply(timestamp, value_scale, in_landmark.z()));
}
return absl::OkStatus();
@@ -165,12 +194,83 @@ class VelocityFilter : public LandmarksFilter {
int window_size_;
float velocity_scale_;
float min_allowed_object_scale_;
bool disable_value_scaling_;
std::vector<RelativeVelocityFilter> x_filters_;
std::vector<RelativeVelocityFilter> y_filters_;
std::vector<RelativeVelocityFilter> z_filters_;
};
// Please check OneEuroFilter documentation for details.
class OneEuroFilterImpl : public LandmarksFilter {
public:
OneEuroFilterImpl(double frequency, double min_cutoff, double beta,
double derivate_cutoff)
: frequency_(frequency),
min_cutoff_(min_cutoff),
beta_(beta),
derivate_cutoff_(derivate_cutoff) {}
absl::Status Reset() override {
x_filters_.clear();
y_filters_.clear();
z_filters_.clear();
return absl::OkStatus();
}
absl::Status Apply(const LandmarkList& in_landmarks,
const absl::Duration& timestamp,
LandmarkList* out_landmarks) override {
// Initialize filters once.
MP_RETURN_IF_ERROR(InitializeFiltersIfEmpty(in_landmarks.landmark_size()));
// Filter landmarks. Every axis of every landmark is filtered separately.
for (int i = 0; i < in_landmarks.landmark_size(); ++i) {
const auto& in_landmark = in_landmarks.landmark(i);
auto* out_landmark = out_landmarks->add_landmark();
*out_landmark = in_landmark;
out_landmark->set_x(x_filters_[i].Apply(timestamp, in_landmark.x()));
out_landmark->set_y(y_filters_[i].Apply(timestamp, in_landmark.y()));
out_landmark->set_z(z_filters_[i].Apply(timestamp, in_landmark.z()));
}
return absl::OkStatus();
}
private:
// Initializes filters for the first time or after Reset. If initialized then
// check the size.
absl::Status InitializeFiltersIfEmpty(const int n_landmarks) {
if (!x_filters_.empty()) {
RET_CHECK_EQ(x_filters_.size(), n_landmarks);
RET_CHECK_EQ(y_filters_.size(), n_landmarks);
RET_CHECK_EQ(z_filters_.size(), n_landmarks);
return absl::OkStatus();
}
for (int i = 0; i < n_landmarks; ++i) {
x_filters_.push_back(
OneEuroFilter(frequency_, min_cutoff_, beta_, derivate_cutoff_));
y_filters_.push_back(
OneEuroFilter(frequency_, min_cutoff_, beta_, derivate_cutoff_));
z_filters_.push_back(
OneEuroFilter(frequency_, min_cutoff_, beta_, derivate_cutoff_));
}
return absl::OkStatus();
}
double frequency_;
double min_cutoff_;
double beta_;
double derivate_cutoff_;
std::vector<OneEuroFilter> x_filters_;
std::vector<OneEuroFilter> y_filters_;
std::vector<OneEuroFilter> z_filters_;
};
} // namespace
// A calculator to smooth landmarks over time.
@@ -207,16 +307,21 @@ class LandmarksSmoothingCalculator : public CalculatorBase {
absl::Status Process(CalculatorContext* cc) override;
private:
LandmarksFilter* landmarks_filter_;
std::unique_ptr<LandmarksFilter> landmarks_filter_;
};
REGISTER_CALCULATOR(LandmarksSmoothingCalculator);
absl::Status LandmarksSmoothingCalculator::GetContract(CalculatorContract* cc) {
cc->Inputs().Tag(kNormalizedLandmarksTag).Set<NormalizedLandmarkList>();
cc->Inputs().Tag(kImageSizeTag).Set<std::pair<int, int>>();
cc->Outputs()
.Tag(kNormalizedFilteredLandmarksTag)
.Set<NormalizedLandmarkList>();
if (cc->Inputs().HasTag(kNormalizedLandmarksTag)) {
cc->Inputs().Tag(kNormalizedLandmarksTag).Set<NormalizedLandmarkList>();
cc->Inputs().Tag(kImageSizeTag).Set<std::pair<int, int>>();
cc->Outputs()
.Tag(kNormalizedFilteredLandmarksTag)
.Set<NormalizedLandmarkList>();
} else {
cc->Inputs().Tag(kLandmarksTag).Set<LandmarkList>();
cc->Outputs().Tag(kFilteredLandmarksTag).Set<LandmarkList>();
}
return absl::OkStatus();
}
@@ -227,12 +332,19 @@ absl::Status LandmarksSmoothingCalculator::Open(CalculatorContext* cc) {
// Pick landmarks filter.
const auto& options = cc->Options<LandmarksSmoothingCalculatorOptions>();
if (options.has_no_filter()) {
landmarks_filter_ = new NoFilter();
landmarks_filter_ = absl::make_unique<NoFilter>();
} else if (options.has_velocity_filter()) {
landmarks_filter_ = new VelocityFilter(
landmarks_filter_ = absl::make_unique<VelocityFilter>(
options.velocity_filter().window_size(),
options.velocity_filter().velocity_scale(),
options.velocity_filter().min_allowed_object_scale());
options.velocity_filter().min_allowed_object_scale(),
options.velocity_filter().disable_value_scaling());
} else if (options.has_one_euro_filter()) {
landmarks_filter_ = absl::make_unique<OneEuroFilterImpl>(
options.one_euro_filter().frequency(),
options.one_euro_filter().min_cutoff(),
options.one_euro_filter().beta(),
options.one_euro_filter().derivate_cutoff());
} else {
RET_CHECK_FAIL()
<< "Landmarks filter is either not specified or not supported";
@@ -244,25 +356,53 @@ absl::Status LandmarksSmoothingCalculator::Open(CalculatorContext* cc) {
absl::Status LandmarksSmoothingCalculator::Process(CalculatorContext* cc) {
// Check that landmarks are not empty and reset the filter if so.
// Don't emit an empty packet for this timestamp.
if (cc->Inputs().Tag(kNormalizedLandmarksTag).IsEmpty()) {
if ((cc->Inputs().HasTag(kNormalizedLandmarksTag) &&
cc->Inputs().Tag(kNormalizedLandmarksTag).IsEmpty()) ||
(cc->Inputs().HasTag(kLandmarksTag) &&
cc->Inputs().Tag(kLandmarksTag).IsEmpty())) {
MP_RETURN_IF_ERROR(landmarks_filter_->Reset());
return absl::OkStatus();
}
const auto& in_landmarks =
cc->Inputs().Tag(kNormalizedLandmarksTag).Get<NormalizedLandmarkList>();
const auto& image_size =
cc->Inputs().Tag(kImageSizeTag).Get<std::pair<int, int>>();
const auto& timestamp =
absl::Microseconds(cc->InputTimestamp().Microseconds());
auto out_landmarks = absl::make_unique<NormalizedLandmarkList>();
MP_RETURN_IF_ERROR(landmarks_filter_->Apply(in_landmarks, image_size,
timestamp, out_landmarks.get()));
if (cc->Inputs().HasTag(kNormalizedLandmarksTag)) {
const auto& in_norm_landmarks =
cc->Inputs().Tag(kNormalizedLandmarksTag).Get<NormalizedLandmarkList>();
cc->Outputs()
.Tag(kNormalizedFilteredLandmarksTag)
.Add(out_landmarks.release(), cc->InputTimestamp());
int image_width;
int image_height;
std::tie(image_width, image_height) =
cc->Inputs().Tag(kImageSizeTag).Get<std::pair<int, int>>();
auto in_landmarks = absl::make_unique<LandmarkList>();
NormalizedLandmarksToLandmarks(in_norm_landmarks, image_width, image_height,
in_landmarks.get());
auto out_landmarks = absl::make_unique<LandmarkList>();
MP_RETURN_IF_ERROR(landmarks_filter_->Apply(*in_landmarks, timestamp,
out_landmarks.get()));
auto out_norm_landmarks = absl::make_unique<NormalizedLandmarkList>();
LandmarksToNormalizedLandmarks(*out_landmarks, image_width, image_height,
out_norm_landmarks.get());
cc->Outputs()
.Tag(kNormalizedFilteredLandmarksTag)
.Add(out_norm_landmarks.release(), cc->InputTimestamp());
} else {
const auto& in_landmarks =
cc->Inputs().Tag(kLandmarksTag).Get<LandmarkList>();
auto out_landmarks = absl::make_unique<LandmarkList>();
MP_RETURN_IF_ERROR(
landmarks_filter_->Apply(in_landmarks, timestamp, out_landmarks.get()));
cc->Outputs()
.Tag(kFilteredLandmarksTag)
.Add(out_landmarks.release(), cc->InputTimestamp());
}
return absl::OkStatus();
}
mediapipe/calculators/util/landmarks_smoothing_calculator.proto
+30
View File
@@ -39,10 +39,40 @@ message LandmarksSmoothingCalculatorOptions {
// If calculated object scale is less than given value smoothing will be
// disabled and landmarks will be returned as is.
optional float min_allowed_object_scale = 3 [default = 1e-6];
// Disable value scaling based on object size and use `1.0` instead.
// Value scale is calculated as inverse value of object size. Object size is
// calculated as maximum side of rectangular bounding box of the object in
// XY plane.
optional bool disable_value_scaling = 4 [default = false];
}
// For the details of the filter implementation and the procedure of its
// configuration please check http://cristal.univ-lille.fr/~casiez/1euro/
message OneEuroFilter {
// Frequency of incomming frames defined in seconds. Used only if can't be
// calculated from provided events (e.g. on the very first frame).
optional float frequency = 1 [default = 0.033];
// Minimum cutoff frequency. Start by tuning this parameter while keeping
// `beta = 0` to reduce jittering to the desired level. 1Hz (the default
// value) is a good starting point.
optional float min_cutoff = 2 [default = 1.0];
// Cutoff slope. After `min_cutoff` is configured, start increasing `beta`
// value to reduce the lag introduced by the `min_cutoff`. Find the desired
// balance between jittering and lag.
optional float beta = 3 [default = 0.0];
// Cutoff frequency for derivate. It is set to 1Hz in the original
// algorithm, but can be tuned to further smooth the speed (i.e. derivate)
// on the object.
optional float derivate_cutoff = 4 [default = 1.0];
}
oneof filter_options {
NoFilter no_filter = 1;
VelocityFilter velocity_filter = 2;
OneEuroFilter one_euro_filter = 3;
}
}
mediapipe/calculators/util/landmarks_to_render_data_calculator.cc
+92 -45
View File
@@ -34,6 +34,34 @@ constexpr char kRenderScaleTag[] = "RENDER_SCALE";
constexpr char kRenderDataTag[] = "RENDER_DATA";
constexpr char kLandmarkLabel[] = "KEYPOINT";
inline Color DefaultMinDepthLineColor() {
Color color;
color.set_r(0);
color.set_g(0);
color.set_b(0);
return color;
}
inline Color DefaultMaxDepthLineColor() {
Color color;
color.set_r(255);
color.set_g(255);
color.set_b(255);
return color;
}
inline Color MixColors(const Color& color1, const Color& color2,
float color1_weight) {
Color color;
color.set_r(static_cast<int>(color1.r() * color1_weight +
color2.r() * (1.f - color1_weight)));
color.set_g(static_cast<int>(color1.g() * color1_weight +
color2.g() * (1.f - color1_weight)));
color.set_b(static_cast<int>(color1.b() * color1_weight +
color2.b() * (1.f - color1_weight)));
return color;
}
inline void SetColor(RenderAnnotation* annotation, const Color& color) {
annotation->mutable_color()->set_r(color.r());
annotation->mutable_color()->set_g(color.g());
@@ -57,6 +85,23 @@ inline void GetMinMaxZ(const LandmarkListType& landmarks, float* z_min,
}
}
template <class LandmarkType>
bool IsLandmarkVisibileAndPresent(const LandmarkType& landmark,
bool utilize_visibility,
float visibility_threshold,
bool utilize_presence,
float presence_threshold) {
if (utilize_visibility && landmark.has_visibility() &&
landmark.visibility() < visibility_threshold) {
return false;
}
if (utilize_presence && landmark.has_presence() &&
landmark.presence() < presence_threshold) {
return false;
}
return true;
}
void SetColorSizeValueFromZ(float z, float z_min, float z_max,
RenderAnnotation* render_annotation,
float min_depth_circle_thickness,
@@ -75,8 +120,9 @@ void SetColorSizeValueFromZ(float z, float z_min, float z_max,
template <class LandmarkType>
void AddConnectionToRenderData(const LandmarkType& start,
const LandmarkType& end, int gray_val1,
int gray_val2, float thickness, bool normalized,
const LandmarkType& end,
const Color& color_start, const Color& color_end,
float thickness, bool normalized,
RenderData* render_data) {
auto* connection_annotation = render_data->add_render_annotations();
RenderAnnotation::GradientLine* line =
@@ -86,12 +132,13 @@ void AddConnectionToRenderData(const LandmarkType& start,
line->set_x_end(end.x());
line->set_y_end(end.y());
line->set_normalized(normalized);
line->mutable_color1()->set_r(gray_val1);
line->mutable_color1()->set_g(gray_val1);
line->mutable_color1()->set_b(gray_val1);
line->mutable_color2()->set_r(gray_val2);
line->mutable_color2()->set_g(gray_val2);
line->mutable_color2()->set_b(gray_val2);
line->mutable_color1()->set_r(color_start.r());
line->mutable_color1()->set_g(color_start.g());
line->mutable_color1()->set_b(color_start.b());
line->mutable_color2()->set_r(color_end.r());
line->mutable_color2()->set_g(color_end.g());
line->mutable_color2()->set_b(color_end.b());
connection_annotation->set_thickness(thickness);
}
@@ -102,26 +149,26 @@ void AddConnectionsWithDepth(const LandmarkListType& landmarks,
float visibility_threshold, bool utilize_presence,
float presence_threshold, float thickness,
bool normalized, float min_z, float max_z,
const Color& min_depth_line_color,
const Color& max_depth_line_color,
RenderData* render_data) {
for (int i = 0; i < landmark_connections.size(); i += 2) {
const auto& ld0 = landmarks.landmark(landmark_connections[i]);
const auto& ld1 = landmarks.landmark(landmark_connections[i + 1]);
if (utilize_visibility &&
((ld0.has_visibility() && ld0.visibility() < visibility_threshold) ||
(ld1.has_visibility() && ld1.visibility() < visibility_threshold))) {
if (!IsLandmarkVisibileAndPresent<LandmarkType>(
ld0, utilize_visibility, visibility_threshold, utilize_presence,
presence_threshold) ||
!IsLandmarkVisibileAndPresent<LandmarkType>(
ld1, utilize_visibility, visibility_threshold, utilize_presence,
presence_threshold)) {
continue;
}
if (utilize_presence &&
((ld0.has_presence() && ld0.presence() < presence_threshold) ||
(ld1.has_presence() && ld1.presence() < presence_threshold))) {
continue;
}
const int gray_val1 =
255 - static_cast<int>(Remap(ld0.z(), min_z, max_z, 255));
const int gray_val2 =
255 - static_cast<int>(Remap(ld1.z(), min_z, max_z, 255));
AddConnectionToRenderData<LandmarkType>(ld0, ld1, gray_val1, gray_val2,
thickness, normalized, render_data);
const Color color0 = MixColors(min_depth_line_color, max_depth_line_color,
Remap(ld0.z(), min_z, max_z, 1.f));
const Color color1 = MixColors(min_depth_line_color, max_depth_line_color,
Remap(ld1.z(), min_z, max_z, 1.f));
AddConnectionToRenderData<LandmarkType>(ld0, ld1, color0, color1, thickness,
normalized, render_data);
}
}
@@ -151,14 +198,12 @@ void AddConnections(const LandmarkListType& landmarks,
for (int i = 0; i < landmark_connections.size(); i += 2) {
const auto& ld0 = landmarks.landmark(landmark_connections[i]);
const auto& ld1 = landmarks.landmark(landmark_connections[i + 1]);
if (utilize_visibility &&
((ld0.has_visibility() && ld0.visibility() < visibility_threshold) ||
(ld1.has_visibility() && ld1.visibility() < visibility_threshold))) {
continue;
}
if (utilize_presence &&
((ld0.has_presence() && ld0.presence() < presence_threshold) ||
(ld1.has_presence() && ld1.presence() < presence_threshold))) {
if (!IsLandmarkVisibileAndPresent<LandmarkType>(
ld0, utilize_visibility, visibility_threshold, utilize_presence,
presence_threshold) ||
!IsLandmarkVisibileAndPresent<LandmarkType>(
ld1, utilize_visibility, visibility_threshold, utilize_presence,
presence_threshold)) {
continue;
}
AddConnectionToRenderData<LandmarkType>(ld0, ld1, connection_color,
@@ -232,6 +277,13 @@ absl::Status LandmarksToRenderDataCalculator::Process(CalculatorContext* cc) {
float z_min = 0.f;
float z_max = 0.f;
const Color min_depth_line_color = options_.has_min_depth_line_color()
? options_.min_depth_line_color()
: DefaultMinDepthLineColor();
const Color max_depth_line_color = options_.has_max_depth_line_color()
? options_.max_depth_line_color()
: DefaultMaxDepthLineColor();
// Apply scale to `thickness` of rendered landmarks and connections to make
// them bigger when object (e.g. pose, hand or face) is closer/bigger and
// snaller when object is further/smaller.
@@ -254,7 +306,7 @@ absl::Status LandmarksToRenderDataCalculator::Process(CalculatorContext* cc) {
landmarks, landmark_connections_, options_.utilize_visibility(),
options_.visibility_threshold(), options_.utilize_presence(),
options_.presence_threshold(), thickness, /*normalized=*/false, z_min,
z_max, render_data.get());
z_max, min_depth_line_color, max_depth_line_color, render_data.get());
} else {
AddConnections<LandmarkList, Landmark>(
landmarks, landmark_connections_, options_.utilize_visibility(),
@@ -265,13 +317,10 @@ absl::Status LandmarksToRenderDataCalculator::Process(CalculatorContext* cc) {
for (int i = 0; i < landmarks.landmark_size(); ++i) {
const Landmark& landmark = landmarks.landmark(i);
if (options_.utilize_visibility() && landmark.has_visibility() &&
landmark.visibility() < options_.visibility_threshold()) {
continue;
}
if (options_.utilize_presence() && landmark.has_presence() &&
landmark.presence() < options_.presence_threshold()) {
if (!IsLandmarkVisibileAndPresent<Landmark>(
landmark, options_.utilize_visibility(),
options_.visibility_threshold(), options_.utilize_presence(),
options_.presence_threshold())) {
continue;
}
@@ -303,7 +352,7 @@ absl::Status LandmarksToRenderDataCalculator::Process(CalculatorContext* cc) {
landmarks, landmark_connections_, options_.utilize_visibility(),
options_.visibility_threshold(), options_.utilize_presence(),
options_.presence_threshold(), thickness, /*normalized=*/true, z_min,
z_max, render_data.get());
z_max, min_depth_line_color, max_depth_line_color, render_data.get());
} else {
AddConnections<NormalizedLandmarkList, NormalizedLandmark>(
landmarks, landmark_connections_, options_.utilize_visibility(),
@@ -314,12 +363,10 @@ absl::Status LandmarksToRenderDataCalculator::Process(CalculatorContext* cc) {
for (int i = 0; i < landmarks.landmark_size(); ++i) {
const NormalizedLandmark& landmark = landmarks.landmark(i);
if (options_.utilize_visibility() && landmark.has_visibility() &&
landmark.visibility() < options_.visibility_threshold()) {
continue;
}
if (options_.utilize_presence() && landmark.has_presence() &&
landmark.presence() < options_.presence_threshold()) {
if (!IsLandmarkVisibileAndPresent<NormalizedLandmark>(
landmark, options_.utilize_visibility(),
options_.visibility_threshold(), options_.utilize_presence(),
options_.presence_threshold())) {
continue;
}
mediapipe/calculators/util/landmarks_to_render_data_calculator.proto
+6
View File
@@ -64,4 +64,10 @@ message LandmarksToRenderDataCalculatorOptions {
// Max thickness of the drawing for landmark circle.
optional double max_depth_circle_thickness = 11 [default = 18.0];
// Gradient color for the lines connecting landmarks at the minimum depth.
optional Color min_depth_line_color = 12;
// Gradient color for the lines connecting landmarks at the maximum depth.
optional Color max_depth_line_color = 13;
}
mediapipe/calculators/util/visibility_copy_calculator.cc
+194
View File
@@ -0,0 +1,194 @@
// Copyright 2021 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <memory>
#include "absl/algorithm/container.h"
#include "mediapipe/calculators/util/visibility_copy_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/landmark.pb.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/timestamp.h"
namespace mediapipe {
namespace {
constexpr char kLandmarksFromTag[] = "LANDMARKS_FROM";
constexpr char kNormalizedLandmarksFromTag[] = "NORM_LANDMARKS_FROM";
constexpr char kLandmarksToTag[] = "LANDMARKS_TO";
constexpr char kNormalizedLandmarksToTag[] = "NORM_LANDMARKS_TO";
} // namespace
// A calculator to copy visibility and presence between landmarks.
//
// Landmarks to copy from and to copy to can be of different type (normalized or
// non-normalized), but ladnmarks to copy to and output landmarks should be of
// the same type. Exactly one stream to copy landmarks from, to copy to and to
// output should be provided.
//
// Inputs:
// LANDMARKS_FROM (optional): A LandmarkList of landmarks to copy from.
// NORM_LANDMARKS_FROM (optional): A NormalizedLandmarkList of landmarks to
// copy from.
// LANDMARKS_TO (optional): A LandmarkList of landmarks to copy to.
// NORM_LANDMARKS_TO (optional): A NormalizedLandmarkList of landmarks to copy
// to.
//
// Outputs:
// LANDMARKS_TO (optional): A LandmarkList of landmarks from LANDMARKS_TO and
// visibility/presence from LANDMARKS_FROM or NORM_LANDMARKS_FROM.
// NORM_LANDMARKS_TO (optional): A NormalizedLandmarkList of landmarks to copy
// to.
//
// Example config:
// node {
// calculator: "VisibilityCopyCalculator"
// input_stream: "NORM_LANDMARKS_FROM:pose_landmarks"
// input_stream: "LANDMARKS_TO:pose_world_landmarks"
// output_stream: "LANDMARKS_TO:pose_world_landmarks_with_visibility"
// options: {
// [mediapipe.VisibilityCopyCalculatorOptions.ext] {
// copy_visibility: true
// copy_presence: true
// }
// }
// }
//
class VisibilityCopyCalculator : public CalculatorBase {
public:
static absl::Status GetContract(CalculatorContract* cc);
absl::Status Open(CalculatorContext* cc) override;
absl::Status Process(CalculatorContext* cc) override;
private:
template <class LandmarkFromType, class LandmarkToType>
absl::Status CopyVisibility(CalculatorContext* cc,
const std::string& landmarks_from_tag,
const std::string& landmarks_to_tag);
bool copy_visibility_;
bool copy_presence_;
};
REGISTER_CALCULATOR(VisibilityCopyCalculator);
absl::Status VisibilityCopyCalculator::GetContract(CalculatorContract* cc) {
// Landmarks to copy from.
RET_CHECK(cc->Inputs().HasTag(kLandmarksFromTag) ^
cc->Inputs().HasTag(kNormalizedLandmarksFromTag))
<< "Exatly one landmarks stream to copy from should be provided";
if (cc->Inputs().HasTag(kLandmarksFromTag)) {
cc->Inputs().Tag(kLandmarksFromTag).Set<LandmarkList>();
} else {
cc->Inputs().Tag(kNormalizedLandmarksFromTag).Set<NormalizedLandmarkList>();
}
// Landmarks to copy to and corresponding output landmarks.
RET_CHECK(cc->Inputs().HasTag(kLandmarksToTag) ^
cc->Inputs().HasTag(kNormalizedLandmarksToTag))
<< "Exatly one landmarks stream to copy to should be provided";
if (cc->Inputs().HasTag(kLandmarksToTag)) {
cc->Inputs().Tag(kLandmarksToTag).Set<LandmarkList>();
RET_CHECK(cc->Outputs().HasTag(kLandmarksToTag))
<< "Landmarks to copy to and output stream types should be the same";
cc->Outputs().Tag(kLandmarksToTag).Set<LandmarkList>();
} else {
cc->Inputs().Tag(kNormalizedLandmarksToTag).Set<NormalizedLandmarkList>();
RET_CHECK(cc->Outputs().HasTag(kNormalizedLandmarksToTag))
<< "Landmarks to copy to and output stream types should be the same";
cc->Outputs().Tag(kNormalizedLandmarksToTag).Set<NormalizedLandmarkList>();
}
return absl::OkStatus();
}
absl::Status VisibilityCopyCalculator::Open(CalculatorContext* cc) {
cc->SetOffset(TimestampDiff(0));
const auto& options = cc->Options<VisibilityCopyCalculatorOptions>();
copy_visibility_ = options.copy_visibility();
copy_presence_ = options.copy_presence();
return absl::OkStatus();
}
absl::Status VisibilityCopyCalculator::Process(CalculatorContext* cc) {
// Switch between all four possible combinations of landmarks from and
// landmarks to types (normalized and non-normalized).
auto status = absl::OkStatus();
if (cc->Inputs().HasTag(kLandmarksFromTag)) {
if (cc->Inputs().HasTag(kLandmarksToTag)) {
status = CopyVisibility<LandmarkList, LandmarkList>(cc, kLandmarksFromTag,
kLandmarksToTag);
} else {
status = CopyVisibility<LandmarkList, NormalizedLandmarkList>(
cc, kLandmarksFromTag, kNormalizedLandmarksToTag);
}
} else {
if (cc->Inputs().HasTag(kLandmarksToTag)) {
status = CopyVisibility<NormalizedLandmarkList, LandmarkList>(
cc, kNormalizedLandmarksFromTag, kLandmarksToTag);
} else {
status = CopyVisibility<NormalizedLandmarkList, NormalizedLandmarkList>(
cc, kNormalizedLandmarksFromTag, kNormalizedLandmarksToTag);
}
}
return status;
}
template <class LandmarkFromType, class LandmarkToType>
absl::Status VisibilityCopyCalculator::CopyVisibility(
CalculatorContext* cc, const std::string& landmarks_from_tag,
const std::string& landmarks_to_tag) {
// Check that both landmarks to copy from and to copy to are non empty.
if (cc->Inputs().Tag(landmarks_from_tag).IsEmpty() ||
cc->Inputs().Tag(landmarks_to_tag).IsEmpty()) {
return absl::OkStatus();
}
const auto landmarks_from =
cc->Inputs().Tag(landmarks_from_tag).Get<LandmarkFromType>();
const auto landmarks_to =
cc->Inputs().Tag(landmarks_to_tag).Get<LandmarkToType>();
auto landmarks_out = absl::make_unique<LandmarkToType>();
for (int i = 0; i < landmarks_from.landmark_size(); ++i) {
const auto& landmark_from = landmarks_from.landmark(i);
const auto& landmark_to = landmarks_to.landmark(i);
// Create output landmark and copy all fields from the `to` landmark.
const auto& landmark_out = landmarks_out->add_landmark();
*landmark_out = landmark_to;
// Copy visibility and presence from the `from` landmark.
if (copy_visibility_) {
landmark_out->set_visibility(landmark_from.visibility());
}
if (copy_presence_) {
landmark_out->set_presence(landmark_from.presence());
}
}
cc->Outputs()
.Tag(landmarks_to_tag)
.Add(landmarks_out.release(), cc->InputTimestamp());
return absl::OkStatus();
}
} // namespace mediapipe
mediapipe/calculators/util/visibility_copy_calculator.proto
+29
View File
@@ -0,0 +1,29 @@
// Copyright 2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto2";
package mediapipe;
import "mediapipe/framework/calculator_options.proto";
message VisibilityCopyCalculatorOptions {
extend CalculatorOptions {
optional VisibilityCopyCalculatorOptions ext = 363728421;
}
optional bool copy_visibility = 1 [default = true];
optional bool copy_presence = 2 [default = true];
}
mediapipe/calculators/util/visibility_smoothing_calculator.cc
+243
View File
@@ -0,0 +1,243 @@
// Copyright 2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <memory>
#include "absl/algorithm/container.h"
#include "mediapipe/calculators/util/visibility_smoothing_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/landmark.pb.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/timestamp.h"
#include "mediapipe/util/filtering/low_pass_filter.h"
namespace mediapipe {
namespace {
constexpr char kNormalizedLandmarksTag[] = "NORM_LANDMARKS";
constexpr char kLandmarksTag[] = "LANDMARKS";
constexpr char kNormalizedFilteredLandmarksTag[] = "NORM_FILTERED_LANDMARKS";
constexpr char kFilteredLandmarksTag[] = "FILTERED_LANDMARKS";
using mediapipe::LowPassFilter;
// Abstract class for various visibility filters.
class VisibilityFilter {
public:
virtual ~VisibilityFilter() = default;
virtual absl::Status Reset() { return absl::OkStatus(); }
virtual absl::Status Apply(const LandmarkList& in_landmarks,
const absl::Duration& timestamp,
LandmarkList* out_landmarks) = 0;
virtual absl::Status Apply(const NormalizedLandmarkList& in_landmarks,
const absl::Duration& timestamp,
NormalizedLandmarkList* out_landmarks) = 0;
};
// Returns visibility as is without smoothing.
class NoFilter : public VisibilityFilter {
public:
absl::Status Apply(const NormalizedLandmarkList& in_landmarks,
const absl::Duration& timestamp,
NormalizedLandmarkList* out_landmarks) override {
*out_landmarks = in_landmarks;
return absl::OkStatus();
}
absl::Status Apply(const LandmarkList& in_landmarks,
const absl::Duration& timestamp,
LandmarkList* out_landmarks) override {
*out_landmarks = in_landmarks;
return absl::OkStatus();
}
};
// Please check LowPassFilter documentation for details.
class LowPassVisibilityFilter : public VisibilityFilter {
public:
LowPassVisibilityFilter(float alpha) : alpha_(alpha) {}
absl::Status Reset() override {
visibility_filters_.clear();
return absl::OkStatus();
}
absl::Status Apply(const LandmarkList& in_landmarks,
const absl::Duration& timestamp,
LandmarkList* out_landmarks) override {
return ApplyImpl<LandmarkList>(in_landmarks, timestamp, out_landmarks);
}
absl::Status Apply(const NormalizedLandmarkList& in_landmarks,
const absl::Duration& timestamp,
NormalizedLandmarkList* out_landmarks) override {
return ApplyImpl<NormalizedLandmarkList>(in_landmarks, timestamp,
out_landmarks);
}
private:
template <class LandmarksType>
absl::Status ApplyImpl(const LandmarksType& in_landmarks,
const absl::Duration& timestamp,
LandmarksType* out_landmarks) {
// Initializes filters for the first time or after Reset. If initialized
// then check the size.
int n_landmarks = in_landmarks.landmark_size();
if (!visibility_filters_.empty()) {
RET_CHECK_EQ(visibility_filters_.size(), n_landmarks);
} else {
visibility_filters_.resize(n_landmarks, LowPassFilter(alpha_));
}
// Filter visibilities.
for (int i = 0; i < in_landmarks.landmark_size(); ++i) {
const auto& in_landmark = in_landmarks.landmark(i);
auto* out_landmark = out_landmarks->add_landmark();
*out_landmark = in_landmark;
out_landmark->set_visibility(
visibility_filters_[i].Apply(in_landmark.visibility()));
}
return absl::OkStatus();
}
float alpha_;
std::vector<LowPassFilter> visibility_filters_;
};
} // namespace
// A calculator to smooth landmark visibilities over time.
//
// Exactly one landmarks input stream is expected. Output stream type should be
// the same as the input one.
//
// Inputs:
// LANDMARKS (optional): A LandmarkList of landmarks you want to smooth.
// NORM_LANDMARKS (optional): A NormalizedLandmarkList of landmarks you want
// to smooth.
//
// Outputs:
// FILTERED_LANDMARKS (optional): A LandmarkList of smoothed landmarks.
// NORM_FILTERED_LANDMARKS (optional): A NormalizedLandmarkList of smoothed
// landmarks.
//
// Example config:
// node {
// calculator: "VisibilitySmoothingCalculator"
// input_stream: "NORM_LANDMARKS:pose_landmarks"
// output_stream: "NORM_FILTERED_LANDMARKS:pose_landmarks_filtered"
// options: {
// [mediapipe.VisibilitySmoothingCalculatorOptions.ext] {
// low_pass_filter: {
// alpha: 0.1
// }
// }
// }
// }
//
class VisibilitySmoothingCalculator : public CalculatorBase {
public:
static absl::Status GetContract(CalculatorContract* cc);
absl::Status Open(CalculatorContext* cc) override;
absl::Status Process(CalculatorContext* cc) override;
private:
std::unique_ptr<VisibilityFilter> visibility_filter_;
};
REGISTER_CALCULATOR(VisibilitySmoothingCalculator);
absl::Status VisibilitySmoothingCalculator::GetContract(
CalculatorContract* cc) {
RET_CHECK(cc->Inputs().HasTag(kNormalizedLandmarksTag) ^
cc->Inputs().HasTag(kLandmarksTag))
<< "Exactly one landmarks input stream is expected";
if (cc->Inputs().HasTag(kNormalizedLandmarksTag)) {
cc->Inputs().Tag(kNormalizedLandmarksTag).Set<NormalizedLandmarkList>();
RET_CHECK(cc->Outputs().HasTag(kNormalizedFilteredLandmarksTag))
<< "Landmarks output stream should of the same type as input one";
cc->Outputs()
.Tag(kNormalizedFilteredLandmarksTag)
.Set<NormalizedLandmarkList>();
} else {
cc->Inputs().Tag(kLandmarksTag).Set<LandmarkList>();
RET_CHECK(cc->Outputs().HasTag(kFilteredLandmarksTag))
<< "Landmarks output stream should of the same type as input one";
cc->Outputs().Tag(kFilteredLandmarksTag).Set<LandmarkList>();
}
return absl::OkStatus();
}
absl::Status VisibilitySmoothingCalculator::Open(CalculatorContext* cc) {
cc->SetOffset(TimestampDiff(0));
// Pick visibility filter.
const auto& options = cc->Options<VisibilitySmoothingCalculatorOptions>();
if (options.has_no_filter()) {
visibility_filter_ = absl::make_unique<NoFilter>();
} else if (options.has_low_pass_filter()) {
visibility_filter_ = absl::make_unique<LowPassVisibilityFilter>(
options.low_pass_filter().alpha());
} else {
RET_CHECK_FAIL()
<< "Visibility filter is either not specified or not supported";
}
return absl::OkStatus();
}
absl::Status VisibilitySmoothingCalculator::Process(CalculatorContext* cc) {
// Check that landmarks are not empty and reset the filter if so.
// Don't emit an empty packet for this timestamp.
if ((cc->Inputs().HasTag(kNormalizedLandmarksTag) &&
cc->Inputs().Tag(kNormalizedLandmarksTag).IsEmpty()) ||
(cc->Inputs().HasTag(kLandmarksTag) &&
cc->Inputs().Tag(kLandmarksTag).IsEmpty())) {
MP_RETURN_IF_ERROR(visibility_filter_->Reset());
return absl::OkStatus();
}
const auto& timestamp =
absl::Microseconds(cc->InputTimestamp().Microseconds());
if (cc->Inputs().HasTag(kNormalizedLandmarksTag)) {
const auto& in_landmarks =
cc->Inputs().Tag(kNormalizedLandmarksTag).Get<NormalizedLandmarkList>();
auto out_landmarks = absl::make_unique<NormalizedLandmarkList>();
MP_RETURN_IF_ERROR(visibility_filter_->Apply(in_landmarks, timestamp,
out_landmarks.get()));
cc->Outputs()
.Tag(kNormalizedFilteredLandmarksTag)
.Add(out_landmarks.release(), cc->InputTimestamp());
} else {
const auto& in_landmarks =
cc->Inputs().Tag(kLandmarksTag).Get<LandmarkList>();
auto out_landmarks = absl::make_unique<LandmarkList>();
MP_RETURN_IF_ERROR(visibility_filter_->Apply(in_landmarks, timestamp,
out_landmarks.get()));
cc->Outputs()
.Tag(kFilteredLandmarksTag)
.Add(out_landmarks.release(), cc->InputTimestamp());
}
return absl::OkStatus();
}
} // namespace mediapipe
mediapipe/calculators/util/visibility_smoothing_calculator.proto
+40
View File
@@ -0,0 +1,40 @@
// Copyright 2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto2";
package mediapipe;
import "mediapipe/framework/calculator_options.proto";
message VisibilitySmoothingCalculatorOptions {
extend CalculatorOptions {
optional VisibilitySmoothingCalculatorOptions ext = 360207350;
}
// Default behaviour and fast way to disable smoothing.
message NoFilter {}
message LowPassFilter {
// Coefficient applied to a new value, whilte `1 - alpha` is applied to a
// stored value. Should be in [0, 1] range. The smaller the value - the
// smoother result and the bigger lag.
optional float alpha = 1 [default = 0.1];
}
oneof filter_options {
NoFilter no_filter = 1;
LowPassFilter low_pass_filter = 2;
}
}
mediapipe/calculators/util/world_landmark_projection_calculator.cc
+108
View File
@@ -0,0 +1,108 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <cmath>
#include <vector>
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/landmark.pb.h"
#include "mediapipe/framework/formats/rect.pb.h"
#include "mediapipe/framework/port/ret_check.h"
namespace mediapipe {
namespace {
constexpr char kLandmarksTag[] = "LANDMARKS";
constexpr char kRectTag[] = "NORM_RECT";
} // namespace
// Projects world landmarks from the rectangle to original coordinates.
//
// World landmarks are predicted in meters rather than in pixels of the image
// and have origin in the middle of the hips rather than in the corner of the
// pose image (cropped with given rectangle). Thus only rotation (but not scale
// and translation) is applied to the landmarks to transform them back to
// original coordinates.
//
// Input:
// LANDMARKS: A LandmarkList representing world landmarks in the rectangle.
// NORM_RECT: An NormalizedRect representing a normalized rectangle in image
// coordinates.
//
// Output:
// LANDMARKS: A LandmarkList representing world landmarks projected (rotated
// but not scaled or translated) from the rectangle to original
// coordinates.
//
// Usage example:
// node {
// calculator: "WorldLandmarkProjectionCalculator"
// input_stream: "LANDMARKS:landmarks"
// input_stream: "NORM_RECT:rect"
// output_stream: "LANDMARKS:projected_landmarks"
// }
//
class WorldLandmarkProjectionCalculator : public CalculatorBase {
public:
static absl::Status GetContract(CalculatorContract* cc) {
cc->Inputs().Tag(kLandmarksTag).Set<LandmarkList>();
cc->Inputs().Tag(kRectTag).Set<NormalizedRect>();
cc->Outputs().Tag(kLandmarksTag).Set<LandmarkList>();
return absl::OkStatus();
}
absl::Status Open(CalculatorContext* cc) override {
cc->SetOffset(TimestampDiff(0));
return absl::OkStatus();
}
absl::Status Process(CalculatorContext* cc) override {
// Check that landmarks and rect are not empty.
if (cc->Inputs().Tag(kLandmarksTag).IsEmpty() ||
cc->Inputs().Tag(kRectTag).IsEmpty()) {
return absl::OkStatus();
}
const auto& in_landmarks =
cc->Inputs().Tag(kLandmarksTag).Get<LandmarkList>();
const auto& in_rect = cc->Inputs().Tag(kRectTag).Get<NormalizedRect>();
auto out_landmarks = absl::make_unique<LandmarkList>();
for (int i = 0; i < in_landmarks.landmark_size(); ++i) {
const auto& in_landmark = in_landmarks.landmark(i);
Landmark* out_landmark = out_landmarks->add_landmark();
*out_landmark = in_landmark;
const float angle = in_rect.rotation();
out_landmark->set_x(std::cos(angle) * in_landmark.x() -
std::sin(angle) * in_landmark.y());
out_landmark->set_y(std::sin(angle) * in_landmark.x() +
std::cos(angle) * in_landmark.y());
}
cc->Outputs()
.Tag(kLandmarksTag)
.Add(out_landmarks.release(), cc->InputTimestamp());
return absl::OkStatus();
}
};
REGISTER_CALCULATOR(WorldLandmarkProjectionCalculator);
} // namespace mediapipe
mediapipe/calculators/video/BUILD
+3
View File
@@ -426,6 +426,7 @@ cc_test(
"//mediapipe/framework/port:logging",
"//mediapipe/framework/port:opencv_core",
"//mediapipe/framework/port:parse_text_proto",
"@com_google_absl//absl/flags:flag",
],
)
@@ -450,6 +451,7 @@ cc_test(
"//mediapipe/framework/port:opencv_imgproc",
"//mediapipe/framework/port:opencv_video",
"//mediapipe/framework/port:parse_text_proto",
"@com_google_absl//absl/flags:flag",
],
)
@@ -534,6 +536,7 @@ cc_test(
"//mediapipe/framework/stream_handler:sync_set_input_stream_handler",
"//mediapipe/util/tracking:box_tracker_cc_proto",
"//mediapipe/util/tracking:tracking_cc_proto",
"@com_google_absl//absl/flags:flag",
],
)