talksik/mediapipe
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Project import generated by Copybara.

Browse Source 
GitOrigin-RevId: 4cee4a2c2317fb190680c17e31ebbb03bb73b71c
This commit is contained in:
MediaPipe Team
2020-09-17 11:09:17 -04:00
committed by chuoling
parent 1db91b550a
commit a908d668c7
142 changed files with 40878 additions and 574 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mediapipe/calculators/core/concatenate_vector_calculator.cc
+3
View File
@@ -50,6 +50,9 @@ REGISTER_CALCULATOR(ConcatenateInt32VectorCalculator);
typedef ConcatenateVectorCalculator<uint64> ConcatenateUInt64VectorCalculator;
REGISTER_CALCULATOR(ConcatenateUInt64VectorCalculator);
typedef ConcatenateVectorCalculator<bool> ConcatenateBoolVectorCalculator;
REGISTER_CALCULATOR(ConcatenateBoolVectorCalculator);
// Example config:
// node {
// calculator: "ConcatenateTfLiteTensorVectorCalculator"
mediapipe/calculators/core/mux_calculator_test.cc
+68
View File
@@ -12,6 +12,8 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include <memory>
#include "mediapipe/calculators/core/split_vector_calculator.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_runner.h"
@@ -233,5 +235,71 @@ TEST(MuxCalculatorTest, InputStreamSelector_MuxInputStreamHandler) {
kOutputName, output_fn);
EXPECT_EQ(output, input_packets);
}
constexpr char kDualInputGraphConfig[] = R"proto(
input_stream: "input_0"
input_stream: "input_1"
input_stream: "input_select"
output_stream: "test_output"
node {
calculator: "MuxCalculator"
input_stream: "INPUT:0:input_0"
input_stream: "INPUT:1:input_1"
input_stream: "SELECT:input_select"
output_stream: "OUTPUT:test_output"
}
)proto";
TEST(MuxCalculatorTest, DiscardSkippedInputs_MuxInputStreamHandler) {
CalculatorGraphConfig config =
::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
kDualInputGraphConfig);
CalculatorGraph graph;
MP_ASSERT_OK(graph.Initialize(config));
std::shared_ptr<int> output;
MP_ASSERT_OK(
graph.ObserveOutputStream("test_output", [&output](const Packet& p) {
output = p.Get<std::shared_ptr<int>>();
return ::mediapipe::OkStatus();
}));
MP_ASSERT_OK(graph.StartRun({}));
auto one = std::make_shared<int>(1);
auto two = std::make_shared<int>(2);
auto three = std::make_shared<int>(3);
std::weak_ptr<int> one_weak = one;
std::weak_ptr<int> two_weak = two;
MP_ASSERT_OK(graph.AddPacketToInputStream(
"input_0",
MakePacket<std::shared_ptr<int>>(std::move(one)).At(Timestamp(0))));
MP_ASSERT_OK(graph.AddPacketToInputStream(
"input_1",
MakePacket<std::shared_ptr<int>>(std::move(two)).At(Timestamp(0))));
MP_ASSERT_OK(graph.AddPacketToInputStream(
"input_1",
MakePacket<std::shared_ptr<int>>(std::move(three)).At(Timestamp(1))));
EXPECT_EQ(one, nullptr);
EXPECT_EQ(two, nullptr);
EXPECT_EQ(three, nullptr);
MP_ASSERT_OK(graph.AddPacketToInputStream(
"input_select", MakePacket<int>(0).At(Timestamp(0))));
MP_ASSERT_OK(graph.WaitUntilIdle());
EXPECT_EQ(*output, 1);
EXPECT_NE(one_weak.lock(), nullptr);
EXPECT_EQ(two_weak.lock(), nullptr);
MP_ASSERT_OK(graph.AddPacketToInputStream(
"input_select", MakePacket<int>(1).At(Timestamp(1))));
MP_ASSERT_OK(graph.WaitUntilIdle());
EXPECT_EQ(*output, 3);
MP_ASSERT_OK(graph.CloseAllInputStreams());
MP_ASSERT_OK(graph.WaitUntilDone());
}
} // namespace
} // namespace mediapipe
mediapipe/calculators/util/BUILD
+27 -1
View File
@@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library", "mediapipe_proto_library")
licenses(["notice"])
@@ -945,6 +945,31 @@ cc_library(
alwayslink = 1,
)
mediapipe_proto_library(
name = "landmarks_smoothing_calculator_proto",
srcs = ["landmarks_smoothing_calculator.proto"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_options_proto",
],
)
cc_library(
name = "landmarks_smoothing_calculator",
srcs = ["landmarks_smoothing_calculator.cc"],
visibility = ["//visibility:public"],
deps = [
":landmarks_smoothing_calculator_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:timestamp",
"//mediapipe/framework/formats:landmark_cc_proto",
"//mediapipe/framework/port:ret_check",
"//mediapipe/util/filtering:relative_velocity_filter",
"@com_google_absl//absl/algorithm:container",
],
alwayslink = 1,
)
cc_library(
name = "landmarks_to_floats_calculator",
srcs = ["landmarks_to_floats_calculator.cc"],
@@ -1103,6 +1128,7 @@ cc_library(
"//mediapipe/framework/formats:classification_cc_proto",
"//mediapipe/framework/formats:landmark_cc_proto",
"//mediapipe/framework/formats:rect_cc_proto",
"//mediapipe/framework/port:integral_types",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"@com_google_absl//absl/strings",
mediapipe/calculators/util/filter_collection_calculator.cc
+5
View File
@@ -20,9 +20,14 @@
#include "mediapipe/framework/formats/classification.pb.h"
#include "mediapipe/framework/formats/landmark.pb.h"
#include "mediapipe/framework/formats/rect.pb.h"
#include "mediapipe/framework/port/integral_types.h"
namespace mediapipe {
typedef FilterCollectionCalculator<std::vector<uint64>>
FilterUInt64CollectionCalculator;
REGISTER_CALCULATOR(FilterUInt64CollectionCalculator);
typedef FilterCollectionCalculator<std::vector<::mediapipe::NormalizedRect>>
FilterNormalizedRectCollectionCalculator;
REGISTER_CALCULATOR(FilterNormalizedRectCollectionCalculator);
mediapipe/calculators/util/landmarks_smoothing_calculator.cc
+273
View File
@@ -0,0 +1,273 @@
// Copyright 2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "absl/algorithm/container.h"
#include "mediapipe/calculators/util/landmarks_smoothing_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/landmark.pb.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/timestamp.h"
#include "mediapipe/util/filtering/relative_velocity_filter.h"
namespace mediapipe {
namespace {
constexpr char kNormalizedLandmarksTag[] = "NORM_LANDMARKS";
constexpr char kImageSizeTag[] = "IMAGE_SIZE";
constexpr char kNormalizedFilteredLandmarksTag[] = "NORM_FILTERED_LANDMARKS";
using ::mediapipe::RelativeVelocityFilter;
// Estimate object scale to use its inverse value as velocity scale for
// RelativeVelocityFilter. If value will be too small (less than
// `options_.min_allowed_object_scale`) smoothing will be disabled and
// landmarks will be returned as is.
// Object scale is calculated as average between bounding box width and height
// with sides parallel to axis.
float GetObjectScale(const NormalizedLandmarkList& landmarks, int image_width,
int image_height) {
const auto& lm_minmax_x = absl::c_minmax_element(
landmarks.landmark(),
[](const auto& a, const auto& b) { return a.x() < b.x(); });
const float x_min = lm_minmax_x.first->x();
const float x_max = lm_minmax_x.second->x();
const auto& lm_minmax_y = absl::c_minmax_element(
landmarks.landmark(),
[](const auto& a, const auto& b) { return a.y() < b.y(); });
const float y_min = lm_minmax_y.first->y();
const float y_max = lm_minmax_y.second->y();
const float object_width = (x_max - x_min) * image_width;
const float object_height = (y_max - y_min) * image_height;
return (object_width + object_height) / 2.0f;
}
// Abstract class for various landmarks filters.
class LandmarksFilter {
public:
virtual ~LandmarksFilter() = default;
virtual ::mediapipe::Status Reset() { return ::mediapipe::OkStatus(); }
virtual ::mediapipe::Status Apply(const NormalizedLandmarkList& in_landmarks,
const std::pair<int, int>& image_size,
const absl::Duration& timestamp,
NormalizedLandmarkList* out_landmarks) = 0;
};
// Returns landmarks as is without smoothing.
class NoFilter : public LandmarksFilter {
public:
::mediapipe::Status Apply(const NormalizedLandmarkList& in_landmarks,
const std::pair<int, int>& image_size,
const absl::Duration& timestamp,
NormalizedLandmarkList* out_landmarks) override {
*out_landmarks = in_landmarks;
return ::mediapipe::OkStatus();
}
};
// Please check RelativeVelocityFilter documentation for details.
class VelocityFilter : public LandmarksFilter {
public:
VelocityFilter(int window_size, float velocity_scale,
float min_allowed_object_scale)
: window_size_(window_size),
velocity_scale_(velocity_scale),
min_allowed_object_scale_(min_allowed_object_scale) {}
::mediapipe::Status Reset() override {
x_filters_.clear();
y_filters_.clear();
z_filters_.clear();
return ::mediapipe::OkStatus();
}
::mediapipe::Status Apply(const NormalizedLandmarkList& in_landmarks,
const std::pair<int, int>& image_size,
const absl::Duration& timestamp,
NormalizedLandmarkList* out_landmarks) override {
// Get image size.
int image_width;
int image_height;
std::tie(image_width, image_height) = image_size;
// Get value scale as inverse value of the object scale.
// If value is too small smoothing will be disabled and landmarks will be
// returned as is.
const float object_scale =
GetObjectScale(in_landmarks, image_width, image_height);
if (object_scale < min_allowed_object_scale_) {
*out_landmarks = in_landmarks;
return ::mediapipe::OkStatus();
}
const float value_scale = 1.0f / object_scale;
// Initialize filters once.
MP_RETURN_IF_ERROR(InitializeFiltersIfEmpty(in_landmarks.landmark_size()));
// Filter landmarks. Every axis of every landmark is filtered separately.
for (int i = 0; i < in_landmarks.landmark_size(); ++i) {
const NormalizedLandmark& in_landmark = in_landmarks.landmark(i);
NormalizedLandmark* out_landmark = out_landmarks->add_landmark();
out_landmark->set_x(x_filters_[i].Apply(timestamp, value_scale,
in_landmark.x() * image_width) /
image_width);
out_landmark->set_y(y_filters_[i].Apply(timestamp, value_scale,
in_landmark.y() * image_height) /
image_height);
// Scale Z the save was as X (using image width).
out_landmark->set_z(z_filters_[i].Apply(timestamp, value_scale,
in_landmark.z() * image_width) /
image_width);
// Keep visibility as is.
out_landmark->set_visibility(in_landmark.visibility());
}
return ::mediapipe::OkStatus();
}
private:
// Initializes filters for the first time or after Reset. If initialized then
// check the size.
::mediapipe::Status InitializeFiltersIfEmpty(const int n_landmarks) {
if (!x_filters_.empty()) {
RET_CHECK_EQ(x_filters_.size(), n_landmarks);
RET_CHECK_EQ(y_filters_.size(), n_landmarks);
RET_CHECK_EQ(z_filters_.size(), n_landmarks);
return ::mediapipe::OkStatus();
}
x_filters_.resize(n_landmarks,
RelativeVelocityFilter(window_size_, velocity_scale_));
y_filters_.resize(n_landmarks,
RelativeVelocityFilter(window_size_, velocity_scale_));
z_filters_.resize(n_landmarks,
RelativeVelocityFilter(window_size_, velocity_scale_));
return ::mediapipe::OkStatus();
}
int window_size_;
float velocity_scale_;
float min_allowed_object_scale_;
std::vector<RelativeVelocityFilter> x_filters_;
std::vector<RelativeVelocityFilter> y_filters_;
std::vector<RelativeVelocityFilter> z_filters_;
};
} // namespace
// A calculator to smooth landmarks over time.
//
// Inputs:
// NORM_LANDMARKS: A NormalizedLandmarkList of landmarks you want to smooth.
// IMAGE_SIZE: A std::pair<int, int> represention of image width and height.
// Required to perform all computations in absolute coordinates to avoid any
// influence of normalized values.
//
// Outputs:
// NORM_FILTERED_LANDMARKS: A NormalizedLandmarkList of smoothed landmarks.
//
// Example config:
// node {
// calculator: "LandmarksSmoothingCalculator"
// input_stream: "NORM_LANDMARKS:pose_landmarks"
// input_stream: "IMAGE_SIZE:image_size"
// output_stream: "NORM_FILTERED_LANDMARKS:pose_landmarks_filtered"
// options: {
// [mediapipe.LandmarksSmoothingCalculatorOptions.ext] {
// velocity_filter: {
// window_size: 5
// velocity_scale: 10.0
// }
// }
// }
// }
//
class LandmarksSmoothingCalculator : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc);
::mediapipe::Status Open(CalculatorContext* cc) override;
::mediapipe::Status Process(CalculatorContext* cc) override;
private:
LandmarksFilter* landmarks_filter_;
};
REGISTER_CALCULATOR(LandmarksSmoothingCalculator);
::mediapipe::Status LandmarksSmoothingCalculator::GetContract(
CalculatorContract* cc) {
cc->Inputs().Tag(kNormalizedLandmarksTag).Set<NormalizedLandmarkList>();
cc->Inputs().Tag(kImageSizeTag).Set<std::pair<int, int>>();
cc->Outputs()
.Tag(kNormalizedFilteredLandmarksTag)
.Set<NormalizedLandmarkList>();
return ::mediapipe::OkStatus();
}
::mediapipe::Status LandmarksSmoothingCalculator::Open(CalculatorContext* cc) {
cc->SetOffset(TimestampDiff(0));
// Pick landmarks filter.
const auto& options = cc->Options<LandmarksSmoothingCalculatorOptions>();
if (options.has_no_filter()) {
landmarks_filter_ = new NoFilter();
} else if (options.has_velocity_filter()) {
landmarks_filter_ = new VelocityFilter(
options.velocity_filter().window_size(),
options.velocity_filter().velocity_scale(),
options.velocity_filter().min_allowed_object_scale());
} else {
RET_CHECK_FAIL()
<< "Landmarks filter is either not specified or not supported";
}
return ::mediapipe::OkStatus();
}
::mediapipe::Status LandmarksSmoothingCalculator::Process(
CalculatorContext* cc) {
// Check that landmarks are not empty and reset the filter if so.
// Don't emit an empty packet for this timestamp.
if (cc->Inputs().Tag(kNormalizedLandmarksTag).IsEmpty()) {
MP_RETURN_IF_ERROR(landmarks_filter_->Reset());
return ::mediapipe::OkStatus();
}
const auto& in_landmarks =
cc->Inputs().Tag(kNormalizedLandmarksTag).Get<NormalizedLandmarkList>();
const auto& image_size =
cc->Inputs().Tag(kImageSizeTag).Get<std::pair<int, int>>();
const auto& timestamp =
absl::Microseconds(cc->InputTimestamp().Microseconds());
auto out_landmarks = absl::make_unique<NormalizedLandmarkList>();
MP_RETURN_IF_ERROR(landmarks_filter_->Apply(in_landmarks, image_size,
timestamp, out_landmarks.get()));
cc->Outputs()
.Tag(kNormalizedFilteredLandmarksTag)
.Add(out_landmarks.release(), cc->InputTimestamp());
return ::mediapipe::OkStatus();
}
} // namespace mediapipe
mediapipe/calculators/util/landmarks_smoothing_calculator.proto
+48
View File
@@ -0,0 +1,48 @@
// Copyright 2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto2";
package mediapipe;
import "mediapipe/framework/calculator_options.proto";
message LandmarksSmoothingCalculatorOptions {
extend CalculatorOptions {
optional LandmarksSmoothingCalculatorOptions ext = 325671429;
}
// Default behaviour and fast way to disable smoothing.
message NoFilter {}
message VelocityFilter {
// Number of value changes to keep over time.
// Higher value adds to lag and to stability.
optional int32 window_size = 1 [default = 5];
// Scale to apply to the velocity calculated over the given window. With
// higher velocity `low pass filter` weights new values higher.
// Lower value adds to lag and to stability.
optional float velocity_scale = 2 [default = 10.0];
// If calculated object scale is less than given value smoothing will be
// disabled and landmarks will be returned as is.
optional float min_allowed_object_scale = 3 [default = 1e-6];
}
oneof filter_options {
NoFilter no_filter = 1;
VelocityFilter velocity_filter = 2;
}
}
mediapipe/calculators/util/landmarks_to_render_data_calculator.cc
+4 -4
View File
@@ -103,8 +103,8 @@ void AddConnectionsWithDepth(const LandmarkListType& landmarks,
for (int i = 0; i < landmark_connections.size(); i += 2) {
const auto& ld0 = landmarks.landmark(landmark_connections[i]);
const auto& ld1 = landmarks.landmark(landmark_connections[i + 1]);
if (visibility_threshold && (ld0.visibility() < visibility_threshold ||
ld1.visibility() < visibility_threshold)) {
if (utilize_visibility && (ld0.visibility() < visibility_threshold ||
ld1.visibility() < visibility_threshold)) {
continue;
}
const int gray_val1 =
@@ -141,8 +141,8 @@ void AddConnections(const LandmarkListType& landmarks,
for (int i = 0; i < landmark_connections.size(); i += 2) {
const auto& ld0 = landmarks.landmark(landmark_connections[i]);
const auto& ld1 = landmarks.landmark(landmark_connections[i + 1]);
if (visibility_threshold && (ld0.visibility() < visibility_threshold ||
ld1.visibility() < visibility_threshold)) {
if (utilize_visibility && (ld0.visibility() < visibility_threshold ||
ld1.visibility() < visibility_threshold)) {
continue;
}
AddConnectionToRenderData<LandmarkType>(ld0, ld1, connection_color,