talksik/mediapipe
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Project import generated by Copybara.

Browse Source 
GitOrigin-RevId: 9295f8ea2339edb71073695ed4fb3fded2f48c60
This commit is contained in:
MediaPipe Team
2020-08-13 01:32:08 -04:00
committed by chuoling
parent 6b0ab0e012
commit d7c287c4e9
248 changed files with 10356 additions and 5328 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mediapipe/graphs/edge_detection/edge_detection_mobile_gpu.pbtxt
+2 -2
View File
@@ -1,7 +1,7 @@
# MediaPipe graph that performs GPU Sobel edge detection on a live video stream.
# Used in the examples in
# mediapipe/examples/android/src/java/com/mediapipe/apps/skeleton and
# mediapipe/examples/ios/edgedetectiongpu.
# mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:helloworld
# and mediapipe/examples/ios/helloworld.
# Images coming into and out of the graph.
input_stream: "input_video"
mediapipe/graphs/pose_tracking/BUILD
+58
View File
@@ -0,0 +1,58 @@
# Copyright 2020 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
load(
"//mediapipe/framework/tool:mediapipe_graph.bzl",
"mediapipe_binary_graph",
)
licenses(["notice"]) # Apache 2.0
package(default_visibility = ["//visibility:public"])
cc_library(
name = "upper_body_pose_tracking_gpu_deps",
deps = [
"//mediapipe/calculators/core:flow_limiter_calculator",
"//mediapipe/calculators/image:image_properties_calculator",
"//mediapipe/graphs/pose_tracking/calculators:landmarks_smoothing_calculator",
"//mediapipe/graphs/pose_tracking/subgraphs:upper_body_pose_renderer_gpu",
"//mediapipe/modules/pose_landmark:pose_landmark_upper_body_gpu",
],
)
mediapipe_binary_graph(
name = "upper_body_pose_tracking_gpu_binary_graph",
graph = "upper_body_pose_tracking_gpu.pbtxt",
output_name = "upper_body_pose_tracking_gpu.binarypb",
deps = [":upper_body_pose_tracking_gpu_deps"],
)
cc_library(
name = "upper_body_pose_tracking_cpu_deps",
deps = [
"//mediapipe/calculators/core:flow_limiter_calculator",
"//mediapipe/calculators/image:image_properties_calculator",
"//mediapipe/graphs/pose_tracking/calculators:landmarks_smoothing_calculator",
"//mediapipe/graphs/pose_tracking/subgraphs:upper_body_pose_renderer_cpu",
"//mediapipe/modules/pose_landmark:pose_landmark_upper_body_cpu",
],
)
mediapipe_binary_graph(
name = "upper_body_pose_tracking_cpu_binary_graph",
graph = "upper_body_pose_tracking_cpu.pbtxt",
output_name = "upper_body_pose_tracking_cpu.binarypb",
deps = [":upper_body_pose_tracking_cpu_deps"],
)
mediapipe/graphs/pose_tracking/calculators/BUILD
+85
View File
@@ -0,0 +1,85 @@
# Copyright 2020 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library")
licenses(["notice"]) # Apache 2.0
package(default_visibility = ["//visibility:public"])
cc_library(
name = "low_pass_filter",
srcs = ["low_pass_filter.cc"],
hdrs = ["low_pass_filter.h"],
deps = [
"//mediapipe/framework/port:logging",
"@com_google_absl//absl/memory",
],
)
cc_test(
name = "low_pass_filter_test",
srcs = ["low_pass_filter_test.cc"],
deps = [
":low_pass_filter",
"//mediapipe/framework/port:gtest_main",
],
)
cc_library(
name = "relative_velocity_filter",
srcs = ["relative_velocity_filter.cc"],
hdrs = ["relative_velocity_filter.h"],
deps = [
":low_pass_filter",
"//mediapipe/framework/port:logging",
"@com_google_absl//absl/memory",
"@com_google_absl//absl/time",
],
)
cc_test(
name = "relative_velocity_filter_test",
srcs = ["relative_velocity_filter_test.cc"],
deps = [
":relative_velocity_filter",
"//mediapipe/framework/port:gtest_main",
"@com_google_absl//absl/time",
],
)
mediapipe_proto_library(
name = "landmarks_smoothing_calculator_proto",
srcs = ["landmarks_smoothing_calculator.proto"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_proto",
],
)
cc_library(
name = "landmarks_smoothing_calculator",
srcs = ["landmarks_smoothing_calculator.cc"],
visibility = ["//visibility:public"],
deps = [
":landmarks_smoothing_calculator_cc_proto",
":relative_velocity_filter",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:timestamp",
"//mediapipe/framework/formats:landmark_cc_proto",
"//mediapipe/framework/port:ret_check",
"@com_google_absl//absl/algorithm:container",
],
alwayslink = 1,
)
mediapipe/graphs/pose_tracking/calculators/landmarks_smoothing_calculator.cc
+273
View File
@@ -0,0 +1,273 @@
// Copyright 2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "absl/algorithm/container.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/landmark.pb.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/timestamp.h"
#include "mediapipe/graphs/pose_tracking/calculators/landmarks_smoothing_calculator.pb.h"
#include "mediapipe/graphs/pose_tracking/calculators/relative_velocity_filter.h"
namespace mediapipe {
namespace {
constexpr char kNormalizedLandmarksTag[] = "NORM_LANDMARKS";
constexpr char kImageSizeTag[] = "IMAGE_SIZE";
constexpr char kNormalizedFilteredLandmarksTag[] = "NORM_FILTERED_LANDMARKS";
using ::mediapipe::RelativeVelocityFilter;
// Estimate object scale to use its inverse value as velocity scale for
// RelativeVelocityFilter. If value will be too small (less than
// `options_.min_allowed_object_scale`) smoothing will be disabled and
// landmarks will be returned as is.
// Object scale is calculated as average between bounding box width and height
// with sides parallel to axis.
float GetObjectScale(const NormalizedLandmarkList& landmarks, int image_width,
int image_height) {
const auto& [lm_min_x, lm_max_x] = absl::c_minmax_element(
landmarks.landmark(),
[](const auto& a, const auto& b) { return a.x() < b.x(); });
const float x_min = lm_min_x->x();
const float x_max = lm_max_x->x();
const auto& [lm_min_y, lm_max_y] = absl::c_minmax_element(
landmarks.landmark(),
[](const auto& a, const auto& b) { return a.y() < b.y(); });
const float y_min = lm_min_y->y();
const float y_max = lm_max_y->y();
const float object_width = (x_max - x_min) * image_width;
const float object_height = (y_max - y_min) * image_height;
return (object_width + object_height) / 2.0f;
}
// Abstract class for various landmarks filters.
class LandmarksFilter {
public:
virtual ~LandmarksFilter() = default;
virtual ::mediapipe::Status Reset() { return ::mediapipe::OkStatus(); }
virtual ::mediapipe::Status Apply(const NormalizedLandmarkList& in_landmarks,
const std::pair<int, int>& image_size,
const absl::Duration& timestamp,
NormalizedLandmarkList* out_landmarks) = 0;
};
// Returns landmarks as is without smoothing.
class NoFilter : public LandmarksFilter {
public:
::mediapipe::Status Apply(const NormalizedLandmarkList& in_landmarks,
const std::pair<int, int>& image_size,
const absl::Duration& timestamp,
NormalizedLandmarkList* out_landmarks) override {
*out_landmarks = in_landmarks;
return ::mediapipe::OkStatus();
}
};
// Please check RelativeVelocityFilter documentation for details.
class VelocityFilter : public LandmarksFilter {
public:
VelocityFilter(int window_size, float velocity_scale,
float min_allowed_object_scale)
: window_size_(window_size),
velocity_scale_(velocity_scale),
min_allowed_object_scale_(min_allowed_object_scale) {}
::mediapipe::Status Reset() override {
x_filters_.clear();
y_filters_.clear();
z_filters_.clear();
return ::mediapipe::OkStatus();
}
::mediapipe::Status Apply(const NormalizedLandmarkList& in_landmarks,
const std::pair<int, int>& image_size,
const absl::Duration& timestamp,
NormalizedLandmarkList* out_landmarks) override {
// Get image size.
int image_width;
int image_height;
std::tie(image_width, image_height) = image_size;
// Get value scale as inverse value of the object scale.
// If value is too small smoothing will be disabled and landmarks will be
// returned as is.
const float object_scale =
GetObjectScale(in_landmarks, image_width, image_height);
if (object_scale < min_allowed_object_scale_) {
*out_landmarks = in_landmarks;
return ::mediapipe::OkStatus();
}
const float value_scale = 1.0f / object_scale;
// Initialize filters once.
MP_RETURN_IF_ERROR(InitializeFiltersIfEmpty(in_landmarks.landmark_size()));
// Filter landmarks. Every axis of every landmark is filtered separately.
for (int i = 0; i < in_landmarks.landmark_size(); ++i) {
const NormalizedLandmark& in_landmark = in_landmarks.landmark(i);
NormalizedLandmark* out_landmark = out_landmarks->add_landmark();
out_landmark->set_x(x_filters_[i].Apply(timestamp, value_scale,
in_landmark.x() * image_width) /
image_width);
out_landmark->set_y(y_filters_[i].Apply(timestamp, value_scale,
in_landmark.y() * image_height) /
image_height);
// Scale Z the save was as X (using image width).
out_landmark->set_z(z_filters_[i].Apply(timestamp, value_scale,
in_landmark.z() * image_width) /
image_width);
// Keep visibility as is.
out_landmark->set_visibility(in_landmark.visibility());
}
return ::mediapipe::OkStatus();
}
private:
// Initializes filters for the first time or after Reset. If initialized then
// check the size.
::mediapipe::Status InitializeFiltersIfEmpty(const int n_landmarks) {
if (!x_filters_.empty()) {
RET_CHECK_EQ(x_filters_.size(), n_landmarks);
RET_CHECK_EQ(y_filters_.size(), n_landmarks);
RET_CHECK_EQ(z_filters_.size(), n_landmarks);
return ::mediapipe::OkStatus();
}
x_filters_.resize(n_landmarks,
RelativeVelocityFilter(window_size_, velocity_scale_));
y_filters_.resize(n_landmarks,
RelativeVelocityFilter(window_size_, velocity_scale_));
z_filters_.resize(n_landmarks,
RelativeVelocityFilter(window_size_, velocity_scale_));
return ::mediapipe::OkStatus();
}
int window_size_;
float velocity_scale_;
float min_allowed_object_scale_;
std::vector<RelativeVelocityFilter> x_filters_;
std::vector<RelativeVelocityFilter> y_filters_;
std::vector<RelativeVelocityFilter> z_filters_;
};
} // namespace
// A calculator to smooth landmarks over time.
//
// Inputs:
// NORM_LANDMARKS: A NormalizedLandmarkList of landmarks you want to smooth.
// IMAGE_SIZE: A std::pair<int, int> represention of image width and height.
// Required to perform all computations in absolute coordinates to avoid any
// influence of normalized values.
//
// Outputs:
// NORM_FILTERED_LANDMARKS: A NormalizedLandmarkList of smoothed landmarks.
//
// Example config:
// node {
// calculator: "LandmarksSmoothingCalculator"
// input_stream: "NORM_LANDMARKS:pose_landmarks"
// input_stream: "IMAGE_SIZE:image_size"
// output_stream: "NORM_FILTERED_LANDMARKS:pose_landmarks_filtered"
// node_options: {
// [type.googleapis.com/mediapipe.LandmarksSmoothingCalculatorOptions] {
// velocity_filter: {
// window_size: 5
// velocity_scale: 10.0
// }
// }
// }
// }
//
class LandmarksSmoothingCalculator : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc);
::mediapipe::Status Open(CalculatorContext* cc) override;
::mediapipe::Status Process(CalculatorContext* cc) override;
private:
LandmarksFilter* landmarks_filter_;
};
REGISTER_CALCULATOR(LandmarksSmoothingCalculator);
::mediapipe::Status LandmarksSmoothingCalculator::GetContract(
CalculatorContract* cc) {
cc->Inputs().Tag(kNormalizedLandmarksTag).Set<NormalizedLandmarkList>();
cc->Inputs().Tag(kImageSizeTag).Set<std::pair<int, int>>();
cc->Outputs()
.Tag(kNormalizedFilteredLandmarksTag)
.Set<NormalizedLandmarkList>();
return ::mediapipe::OkStatus();
}
::mediapipe::Status LandmarksSmoothingCalculator::Open(CalculatorContext* cc) {
cc->SetOffset(TimestampDiff(0));
// Pick landmarks filter.
const auto& options = cc->Options<LandmarksSmoothingCalculatorOptions>();
if (options.has_no_filter()) {
landmarks_filter_ = new NoFilter();
} else if (options.has_velocity_filter()) {
landmarks_filter_ = new VelocityFilter(
options.velocity_filter().window_size(),
options.velocity_filter().velocity_scale(),
options.velocity_filter().min_allowed_object_scale());
} else {
RET_CHECK_FAIL()
<< "Landmarks filter is either not specified or not supported";
}
return ::mediapipe::OkStatus();
}
::mediapipe::Status LandmarksSmoothingCalculator::Process(
CalculatorContext* cc) {
// Check that landmarks are not empty and reset the filter if so.
// Don't emit an empty packet for this timestamp.
if (cc->Inputs().Tag(kNormalizedLandmarksTag).IsEmpty()) {
MP_RETURN_IF_ERROR(landmarks_filter_->Reset());
return ::mediapipe::OkStatus();
}
const auto& in_landmarks =
cc->Inputs().Tag(kNormalizedLandmarksTag).Get<NormalizedLandmarkList>();
const auto& image_size =
cc->Inputs().Tag(kImageSizeTag).Get<std::pair<int, int>>();
const auto& timestamp =
absl::Microseconds(cc->InputTimestamp().Microseconds());
auto out_landmarks = absl::make_unique<NormalizedLandmarkList>();
MP_RETURN_IF_ERROR(landmarks_filter_->Apply(in_landmarks, image_size,
timestamp, out_landmarks.get()));
cc->Outputs()
.Tag(kNormalizedFilteredLandmarksTag)
.Add(out_landmarks.release(), cc->InputTimestamp());
return ::mediapipe::OkStatus();
}
} // namespace mediapipe
mediapipe/graphs/pose_tracking/calculators/landmarks_smoothing_calculator.proto
+48
View File
@@ -0,0 +1,48 @@
// Copyright 2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto2";
package mediapipe;
import "mediapipe/framework/calculator.proto";
message LandmarksSmoothingCalculatorOptions {
extend CalculatorOptions {
optional LandmarksSmoothingCalculatorOptions ext = 325671429;
}
// Default behaviour and fast way to disable smoothing.
message NoFilter {}
message VelocityFilter {
// Number of value changes to keep over time.
// Higher value adds to lag and to stability.
optional int32 window_size = 1 [default = 5];
// Scale to apply to the velocity calculated over the given window. With
// higher velocity `low pass filter` weights new values higher.
// Lower value adds to lag and to stability.
optional float velocity_scale = 2 [default = 10.0];
// If calculated object scale is less than given value smoothing will be
// disabled and landmarks will be returned as is.
optional float min_allowed_object_scale = 3 [default = 1e-6];
}
oneof filter_options {
NoFilter no_filter = 1;
VelocityFilter velocity_filter = 2;
}
}
mediapipe/graphs/pose_tracking/calculators/low_pass_filter.cc
+58
View File
@@ -0,0 +1,58 @@
// Copyright 2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/graphs/pose_tracking/calculators/low_pass_filter.h"
#include "absl/memory/memory.h"
#include "mediapipe/framework/port/logging.h"
namespace mediapipe {
LowPassFilter::LowPassFilter(float alpha) : initialized_{false} {
SetAlpha(alpha);
}
float LowPassFilter::Apply(float value) {
float result;
if (initialized_) {
result = alpha_ * value + (1.0 - alpha_) * stored_value_;
} else {
result = value;
initialized_ = true;
}
raw_value_ = value;
stored_value_ = result;
return result;
}
float LowPassFilter::ApplyWithAlpha(float value, float alpha) {
SetAlpha(alpha);
return Apply(value);
}
bool LowPassFilter::HasLastRawValue() { return initialized_; }
float LowPassFilter::LastRawValue() { return raw_value_; }
float LowPassFilter::LastValue() { return stored_value_; }
void LowPassFilter::SetAlpha(float alpha) {
if (alpha < 0.0f || alpha > 1.0f) {
LOG(ERROR) << "alpha: " << alpha << " should be in [0.0, 1.0] range";
return;
}
alpha_ = alpha;
}
} // namespace mediapipe
mediapipe/graphs/pose_tracking/calculators/low_pass_filter.h
+47
View File
@@ -0,0 +1,47 @@
// Copyright 2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef MEDIAPIPE_GRAPHS_POSE_TRACKING_CALCULATORS_LOW_PASS_FILTER_H_
#define MEDIAPIPE_GRAPHS_POSE_TRACKING_CALCULATORS_LOW_PASS_FILTER_H_
#include <memory>
namespace mediapipe {
class LowPassFilter {
public:
explicit LowPassFilter(float alpha);
float Apply(float value);
float ApplyWithAlpha(float value, float alpha);
bool HasLastRawValue();
float LastRawValue();
float LastValue();
private:
void SetAlpha(float alpha);
float raw_value_;
float alpha_;
float stored_value_;
bool initialized_;
};
} // namespace mediapipe
#endif // MEDIAPIPE_GRAPHS_POSE_TRACKING_CALCULATORS_LOW_PASS_FILTER_H_
mediapipe/graphs/pose_tracking/calculators/low_pass_filter_test.cc
+35
View File
@@ -0,0 +1,35 @@
// Copyright 2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/graphs/pose_tracking/calculators/low_pass_filter.h"
#include "mediapipe/framework/port/gtest.h"
namespace mediapipe {
TEST(LowPassFilterTest, LowPassFilterBasicChecks) {
auto filter = absl::make_unique<LowPassFilter>(1.0f);
EXPECT_EQ(2.0f, filter->Apply(2.0f));
EXPECT_EQ(100.0f, filter->Apply(100.0f));
filter = absl::make_unique<LowPassFilter>(0.0f);
EXPECT_EQ(2.0f, filter->Apply(2.0f));
EXPECT_EQ(2.0f, filter->Apply(100.0f));
filter = absl::make_unique<LowPassFilter>(0.5f);
EXPECT_EQ(2.0f, filter->Apply(2.0f));
EXPECT_EQ(51.0f, filter->Apply(100.0f));
}
} // namespace mediapipe
mediapipe/graphs/pose_tracking/calculators/relative_velocity_filter.cc
+85
View File
@@ -0,0 +1,85 @@
// Copyright 2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/graphs/pose_tracking/calculators/relative_velocity_filter.h"
#include <cmath>
#include <deque>
#include "absl/memory/memory.h"
#include "mediapipe/framework/port/logging.h"
namespace mediapipe {
float RelativeVelocityFilter::Apply(absl::Duration timestamp, float value_scale,
float value) {
const int64_t new_timestamp = absl::ToInt64Nanoseconds(timestamp);
if (last_timestamp_ >= new_timestamp) {
// Results are unpredictable in this case, so nothing to do but
// return same value
LOG(WARNING) << "New timestamp is equal or less than the last one.";
return value;
}
float alpha;
if (last_timestamp_ == -1) {
alpha = 1.0;
} else {
DCHECK(distance_mode_ == DistanceEstimationMode::kLegacyTransition ||
distance_mode_ == DistanceEstimationMode::kForceCurrentScale);
const float distance =
distance_mode_ == DistanceEstimationMode::kLegacyTransition
? value * value_scale -
last_value_ * last_value_scale_ // Original.
: value_scale * (value - last_value_); // Translation invariant.
const int64_t duration = new_timestamp - last_timestamp_;
float cumulative_distance = distance;
int64_t cumulative_duration = duration;
// Define max cumulative duration assuming
// 30 frames per second is a good frame rate, so assuming 30 values
// per second or 1 / 30 of a second is a good duration per window element
constexpr int64_t kAssumedMaxDuration = 1000000000 / 30;
const int64_t max_cumulative_duration =
(1 + window_.size()) * kAssumedMaxDuration;
for (const auto& el : window_) {
if (cumulative_duration + el.duration > max_cumulative_duration) {
// This helps in cases when durations are large and outdated
// window elements have bad impact on filtering results
break;
}
cumulative_distance += el.distance;
cumulative_duration += el.duration;
}
constexpr double kNanoSecondsToSecond = 1e-9;
const float velocity =
cumulative_distance / (cumulative_duration * kNanoSecondsToSecond);
alpha = 1.0f - 1.0f / (1.0f + velocity_scale_ * std::abs(velocity));
window_.push_front({distance, duration});
if (window_.size() > max_window_size_) {
window_.pop_back();
}
}
last_value_ = value;
last_value_scale_ = value_scale;
last_timestamp_ = new_timestamp;
return low_pass_filter_.ApplyWithAlpha(value, alpha);
}
} // namespace mediapipe
mediapipe/graphs/pose_tracking/calculators/relative_velocity_filter.h
+90
View File
@@ -0,0 +1,90 @@
// Copyright 2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef MEDIAPIPE_GRAPHS_POSE_TRACKING_CALCULATORS_RELATIVE_VELOCITY_FILTER_H_
#define MEDIAPIPE_GRAPHS_POSE_TRACKING_CALCULATORS_RELATIVE_VELOCITY_FILTER_H_
#include <deque>
#include <memory>
#include "absl/time/time.h"
#include "mediapipe/graphs/pose_tracking/calculators/low_pass_filter.h"
namespace mediapipe {
// This filter keeps track (on a window of specified size) of
// value changes over time, which as result gives us velocity of how value
// changes over time. With higher velocity it weights new values higher.
//
// Use @window_size and @velocity_scale to tweak this filter for your use case.
//
// - higher @window_size adds to lag and to stability
// - lower @velocity_scale adds to lag and to stability
class RelativeVelocityFilter {
public:
enum class DistanceEstimationMode {
// When the value scale changes, uses a heuristic
// that is not translation invariant (see the implementation for details).
kLegacyTransition,
// The current (i.e. last) value scale is always used for scale estimation.
// When using this mode, the filter is translation invariant, i.e.
// Filter(Data + Offset) = Filter(Data) + Offset.
kForceCurrentScale,
kDefault = kLegacyTransition
};
public:
RelativeVelocityFilter(size_t window_size, float velocity_scale,
DistanceEstimationMode distance_mode)
: max_window_size_{window_size},
window_{window_size},
velocity_scale_{velocity_scale},
distance_mode_{distance_mode} {}
RelativeVelocityFilter(size_t window_size, float velocity_scale)
: RelativeVelocityFilter{window_size, velocity_scale,
DistanceEstimationMode::kDefault} {}
// Applies filter to the value.
// @timestamp - timestamp associated with the value (for instance,
// timestamp of the frame where you got value from)
// @value_scale - value scale (for instance, if your value is a distance
// detected on a frame, it can look same on different
// devices but have quite different absolute values due
// to different resolution, you should come up with an
// appropriate parameter for your particular use case)
// @value - value to filter
float Apply(absl::Duration timestamp, float value_scale, float value);
private:
struct WindowElement {
float distance;
int64_t duration;
};
float last_value_{0.0};
float last_value_scale_{1.0};
int64_t last_timestamp_{-1};
size_t max_window_size_;
std::deque<WindowElement> window_;
LowPassFilter low_pass_filter_{1.0f};
float velocity_scale_;
DistanceEstimationMode distance_mode_;
};
} // namespace mediapipe
#endif // MEDIAPIPE_GRAPHS_POSE_TRACKING_CALCULATORS_RELATIVE_VELOCITY_FILTER_H_
mediapipe/graphs/pose_tracking/calculators/relative_velocity_filter_test.cc
+292
View File
@@ -0,0 +1,292 @@
// Copyright 2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/graphs/pose_tracking/calculators/relative_velocity_filter.h"
#include <algorithm>
#include <cmath>
#include <vector>
#include "absl/time/time.h"
#include "mediapipe/framework/port/gtest.h"
namespace mediapipe {
using DistanceEstimationMode =
::mediapipe::RelativeVelocityFilter::DistanceEstimationMode;
absl::Duration DurationFromNanos(int64_t nanos) {
return absl::FromChrono(std::chrono::nanoseconds{nanos});
}
absl::Duration DurationFromMillis(int64_t millis) {
return absl::FromChrono(std::chrono::milliseconds{millis});
}
TEST(RelativeVelocityFilterTest, ApplyIncorrectTimestamp) {
auto filter = absl::make_unique<RelativeVelocityFilter>(1, 1.0);
absl::Duration timestamp1 = DurationFromNanos(1);
EXPECT_FLOAT_EQ(95.5f, filter->Apply(timestamp1, 0.5f, 95.5f));
EXPECT_FLOAT_EQ(200.5f, filter->Apply(timestamp1, 0.5f, 200.5f));
EXPECT_FLOAT_EQ(1000.5f, filter->Apply(timestamp1, 0.5f, 1000.5f));
EXPECT_FLOAT_EQ(2000.0f, filter->Apply(DurationFromNanos(1), 0.5f, 2000.0f));
}
void TestSameValueScaleDifferentVelocityScales(
DistanceEstimationMode distance_mode) {
// Changing the distance estimation mode has no effect with constant scales.
// More sensitive filter.
auto filter1 = absl::make_unique<RelativeVelocityFilter>(
/*window_size=*/5, /*velocity_scale=*/45.0f,
/*distance_mode=*/distance_mode);
// Less sensitive filter.
auto filter2 = absl::make_unique<RelativeVelocityFilter>(
/*window_size=*/5, /*velocity_scale=*/0.1f,
/*distance_mode=*/distance_mode);
float result1;
float result2;
float value;
float value_scale = 1.0f;
value = 1.0f;
result1 = filter1->Apply(DurationFromMillis(1), value_scale, value);
result2 = filter2->Apply(DurationFromMillis(1), value_scale, value);
EXPECT_EQ(result1, result2);
value = 10.0f;
result1 = filter1->Apply(DurationFromMillis(2), value_scale, value);
result2 = filter2->Apply(DurationFromMillis(2), value_scale, value);
EXPECT_GT(result1, result2);
value = 2.0f;
result1 = filter1->Apply(DurationFromMillis(3), value_scale, value);
result2 = filter2->Apply(DurationFromMillis(3), value_scale, value);
EXPECT_LT(result1, result2);
value = 20.0f;
result1 = filter1->Apply(DurationFromMillis(4), value_scale, value);
result2 = filter2->Apply(DurationFromMillis(4), value_scale, value);
EXPECT_GT(result1, result2);
value = 10.0f;
result1 = filter1->Apply(DurationFromMillis(5), value_scale, value);
result2 = filter2->Apply(DurationFromMillis(5), value_scale, value);
EXPECT_LT(result1, result2);
value = 50.0f;
result1 = filter1->Apply(DurationFromMillis(6), value_scale, value);
result2 = filter2->Apply(DurationFromMillis(6), value_scale, value);
EXPECT_GT(result1, result2);
value = 30.0f;
result1 = filter1->Apply(DurationFromMillis(7), value_scale, value);
result2 = filter2->Apply(DurationFromMillis(7), value_scale, value);
EXPECT_LT(result1, result2);
}
TEST(RelativeVelocityFilterTest, SameValueScaleDifferentVelocityScalesLegacy) {
TestSameValueScaleDifferentVelocityScales(
DistanceEstimationMode::kLegacyTransition);
}
TEST(RelativeVelocityFilterTest,
SameValueScaleDifferentVelocityScalesForceCurrentScale) {
TestSameValueScaleDifferentVelocityScales(
DistanceEstimationMode::kForceCurrentScale);
}
void TestDifferentConstantValueScalesSameVelocityScale(
DistanceEstimationMode distance_mode) {
const float same_velocity_scale = 1.0f;
auto filter1 = absl::make_unique<RelativeVelocityFilter>(
/*window_size=*/3, /*velocity_scale=*/same_velocity_scale,
/*distance_mode=*/distance_mode);
auto filter2 = absl::make_unique<RelativeVelocityFilter>(
/*window_size=*/3, /*velocity_scale=*/same_velocity_scale,
/*distance_mode=*/distance_mode);
float result1;
float result2;
float value;
// smaller value scale will decrease cumulative speed and alpha
// so with smaller scale and same other params filter will believe
// new values a little bit less
float value_scale1 = 0.5f;
float value_scale2 = 1.0f;
value = 1.0f;
result1 = filter1->Apply(DurationFromMillis(1), value_scale1, value);
result2 = filter2->Apply(DurationFromMillis(1), value_scale2, value);
EXPECT_EQ(result1, result2);
value = 10.0f;
result1 = filter1->Apply(DurationFromMillis(2), value_scale1, value);
result2 = filter2->Apply(DurationFromMillis(2), value_scale2, value);
EXPECT_LT(result1, result2);
value = 2.0f;
result1 = filter1->Apply(DurationFromMillis(3), value_scale1, value);
result2 = filter2->Apply(DurationFromMillis(3), value_scale2, value);
EXPECT_GT(result1, result2);
value = 20.0f;
result1 = filter1->Apply(DurationFromMillis(4), value_scale1, value);
result2 = filter2->Apply(DurationFromMillis(4), value_scale2, value);
EXPECT_LT(result1, result2);
}
TEST(RelativeVelocityFilterTest,
DifferentConstantValueScalesSameVelocityScale) {
TestDifferentConstantValueScalesSameVelocityScale(
DistanceEstimationMode::kLegacyTransition);
}
TEST(RelativeVelocityFilterTest, ApplyCheckValueScales) {
TestDifferentConstantValueScalesSameVelocityScale(
DistanceEstimationMode::kForceCurrentScale);
}
void TestTranslationInvariance(DistanceEstimationMode distance_mode) {
struct ValueAtScale {
float value;
float scale;
};
// Note that the scales change over time.
std::vector<ValueAtScale> original_data_points{
// clang-format off
{.value = 1.0f, .scale = 0.5f},
{.value = 10.0f, .scale = 5.0f},
{.value = 20.0f, .scale = 10.0f},
{.value = 30.0f, .scale = 15.0f},
{.value = 40.0f, .scale = 0.5f},
{.value = 50.0f, .scale = 0.5f},
{.value = 60.0f, .scale = 5.0f},
{.value = 70.0f, .scale = 10.0f},
{.value = 80.0f, .scale = 15.0f},
{.value = 90.0f, .scale = 5.0f},
{.value = 70.0f, .scale = 10.0f},
{.value = 50.0f, .scale = 15.0f},
{.value = 80.0f, .scale = 15.0f},
// clang-format on
};
// The amount by which the input values are uniformly translated.
const float kValueOffset = 100.0f;
// The uniform time delta.
const absl::Duration time_delta = DurationFromMillis(1);
// The filter parameters are the same between the two filters.
const size_t kWindowSize = 5;
const float kVelocityScale = 0.1f;
// Perform the translation.
std::vector<ValueAtScale> translated_data_points = original_data_points;
for (auto& point : translated_data_points) {
point.value += kValueOffset;
}
auto original_points_filter = absl::make_unique<RelativeVelocityFilter>(
/*window_size=*/kWindowSize, /*velocity_scale=*/kVelocityScale,
/*distance_mode=*/distance_mode);
auto translated_points_filter = absl::make_unique<RelativeVelocityFilter>(
/*window_size=*/kWindowSize, /*velocity_scale=*/kVelocityScale,
/*distance_mode=*/distance_mode);
// The minimal difference which is considered a divergence.
const float kDivergenceGap = 0.001f;
// The amount of the times this gap is achieved with `kLegacyTransition`.
// Note that on the first iteration the filters should output the unfiltered
// input values, so no divergence should occur.
// This amount obviously depends on the values in `original_data_points`,
// so should be changed accordingly when they are updated.
const size_t kDivergenceTimes = 5;
// The minimal difference which is considered a large divergence.
const float kLargeDivergenceGap = 10.0f;
// The amount of times it is achieved.
// This amount obviously depends on the values in `original_data_points`,
// so should be changed accordingly when they are updated.
const size_t kLargeDivergenceTimes = 1;
// In contrast, the new mode delivers this error bound across all the samples.
const float kForceCurrentScaleAbsoluteError = 1.53e-05f;
size_t times_diverged = 0;
size_t times_largely_diverged = 0;
absl::Duration timestamp;
for (size_t iteration = 0; iteration < original_data_points.size();
++iteration, timestamp += time_delta) {
const ValueAtScale& original_data_point = original_data_points[iteration];
const float filtered_original_value =
original_points_filter->Apply(/*timestamp=*/timestamp,
/*value_scale=*/original_data_point.scale,
/*value=*/original_data_point.value);
const ValueAtScale& translated_data_point =
translated_data_points[iteration];
const float actual_filtered_translated_value =
translated_points_filter->Apply(
/*timestamp=*/timestamp,
/*value_scale=*/translated_data_point.scale,
/*value=*/translated_data_point.value);
const float expected_filtered_translated_value =
filtered_original_value + kValueOffset;
const float difference = std::fabs(actual_filtered_translated_value -
expected_filtered_translated_value);
if (iteration == 0) {
// On the first iteration, the unfiltered values are returned.
EXPECT_EQ(filtered_original_value, original_data_point.value);
EXPECT_EQ(actual_filtered_translated_value, translated_data_point.value);
EXPECT_EQ(difference, 0.0f);
} else if (distance_mode == DistanceEstimationMode::kLegacyTransition) {
if (difference >= kDivergenceGap) {
++times_diverged;
}
if (difference >= kLargeDivergenceGap) {
++times_largely_diverged;
}
} else {
CHECK(distance_mode == DistanceEstimationMode::kForceCurrentScale);
EXPECT_NEAR(difference, 0.0f, kForceCurrentScaleAbsoluteError);
}
}
if (distance_mode == DistanceEstimationMode::kLegacyTransition) {
EXPECT_GE(times_diverged, kDivergenceTimes);
EXPECT_GE(times_largely_diverged, kLargeDivergenceTimes);
}
}
// This test showcases an undesired property of the current filter design
// that manifests itself when value scales change in time. It turns out that
// the velocity estimation starts depending on the distance from the origin.
TEST(RelativeVelocityFilterTest,
TestLegacyFilterModeIsNotTranslationInvariant) {
TestTranslationInvariance(DistanceEstimationMode::kLegacyTransition);
}
TEST(RelativeVelocityFilterTest, TestOtherFilterModeIsTranslationInvariant) {
TestTranslationInvariance(DistanceEstimationMode::kForceCurrentScale);
}
} // namespace mediapipe
mediapipe/graphs/pose_tracking/subgraphs/BUILD
+50
View File
@@ -0,0 +1,50 @@
# Copyright 2019 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
load(
"//mediapipe/framework/tool:mediapipe_graph.bzl",
"mediapipe_simple_subgraph",
)
licenses(["notice"]) # Apache 2.0
package(default_visibility = ["//visibility:public"])
mediapipe_simple_subgraph(
name = "upper_body_pose_renderer_gpu",
graph = "upper_body_pose_renderer_gpu.pbtxt",
register_as = "UpperBodyPoseRendererGpu",
deps = [
"//mediapipe/calculators/core:split_normalized_landmark_list_calculator",
"//mediapipe/calculators/util:annotation_overlay_calculator",
"//mediapipe/calculators/util:detections_to_render_data_calculator",
"//mediapipe/calculators/util:landmarks_to_render_data_calculator",
"//mediapipe/calculators/util:rect_to_render_data_calculator",
"//mediapipe/calculators/util:rect_to_render_scale_calculator",
],
)
mediapipe_simple_subgraph(
name = "upper_body_pose_renderer_cpu",
graph = "upper_body_pose_renderer_cpu.pbtxt",
register_as = "UpperBodyPoseRendererCpu",
deps = [
"//mediapipe/calculators/core:split_normalized_landmark_list_calculator",
"//mediapipe/calculators/util:annotation_overlay_calculator",
"//mediapipe/calculators/util:detections_to_render_data_calculator",
"//mediapipe/calculators/util:landmarks_to_render_data_calculator",
"//mediapipe/calculators/util:rect_to_render_data_calculator",
"//mediapipe/calculators/util:rect_to_render_scale_calculator",
],
)
mediapipe/graphs/pose_tracking/subgraphs/upper_body_pose_renderer_cpu.pbtxt
+254
View File
@@ -0,0 +1,254 @@
# MediaPipe pose landmarks rendering subgraph.
type: "UpperBodyPoseRendererCpu"
# CPU image. (ImageFrame)
input_stream: "IMAGE:input_image"
# Pose landmarks. (NormalizedLandmarkList)
input_stream: "LANDMARKS:pose_landmarks"
# Region of interest calculated based on landmarks. (NormalizedRect)
input_stream: "ROI:roi"
# Detected pose. (Detection)
input_stream: "DETECTION:detection"
# CPU image with rendered data. (ImageFrame)
output_stream: "IMAGE:output_image"
node {
calculator: "ImagePropertiesCalculator"
input_stream: "IMAGE:input_image"
output_stream: "SIZE:image_size"
}
# Calculates rendering scale based on the pose roi.
node {
calculator: "RectToRenderScaleCalculator"
input_stream: "NORM_RECT:roi"
input_stream: "IMAGE_SIZE:image_size"
output_stream: "RENDER_SCALE:render_scale"
node_options: {
[type.googleapis.com/mediapipe.RectToRenderScaleCalculatorOptions] {
multiplier: 0.0012
}
}
}
# Converts detections to drawing primitives for annotation overlay.
node {
calculator: "DetectionsToRenderDataCalculator"
input_stream: "DETECTION:detection"
output_stream: "RENDER_DATA:detection_render_data"
node_options: {
[type.googleapis.com/mediapipe.DetectionsToRenderDataCalculatorOptions] {
thickness: 4.0
color { r: 0 g: 255 b: 0 }
}
}
}
node {
calculator: "SplitNormalizedLandmarkListCalculator"
input_stream: "pose_landmarks"
output_stream: "visible_pose_landmarks"
node_options: {
[type.googleapis.com/mediapipe.SplitVectorCalculatorOptions] {
ranges: { begin: 0 end: 25 }
}
}
}
# Converts landmarks to drawing primitives for annotation overlay.
node {
calculator: "LandmarksToRenderDataCalculator"
input_stream: "NORM_LANDMARKS:pose_landmarks"
input_stream: "RENDER_SCALE:render_scale"
output_stream: "RENDER_DATA:landmarks_render_data"
node_options: {
[type.googleapis.com/mediapipe.LandmarksToRenderDataCalculatorOptions] {
landmark_connections: 0
landmark_connections: 1
landmark_connections: 1
landmark_connections: 2
landmark_connections: 2
landmark_connections: 3
landmark_connections: 3
landmark_connections: 7
landmark_connections: 0
landmark_connections: 4
landmark_connections: 4
landmark_connections: 5
landmark_connections: 5
landmark_connections: 6
landmark_connections: 6
landmark_connections: 8
landmark_connections: 9
landmark_connections: 10
landmark_connections: 11
landmark_connections: 12
landmark_connections: 11
landmark_connections: 13
landmark_connections: 13
landmark_connections: 15
landmark_connections: 15
landmark_connections: 17
landmark_connections: 15
landmark_connections: 19
landmark_connections: 15
landmark_connections: 21
landmark_connections: 17
landmark_connections: 19
landmark_connections: 12
landmark_connections: 14
landmark_connections: 14
landmark_connections: 16
landmark_connections: 16
landmark_connections: 18
landmark_connections: 16
landmark_connections: 20
landmark_connections: 16
landmark_connections: 22
landmark_connections: 18
landmark_connections: 20
landmark_connections: 11
landmark_connections: 23
landmark_connections: 12
landmark_connections: 24
landmark_connections: 23
landmark_connections: 24
landmark_color { r: 255 g: 255 b: 255 }
connection_color { r: 255 g: 255 b: 255 }
thickness: 3.0
visualize_landmark_depth: false
utilize_visibility: true
visibility_threshold: 0.1
}
}
}
# Take left pose landmarks.
node {
calculator: "SplitNormalizedLandmarkListCalculator"
input_stream: "pose_landmarks"
output_stream: "landmarks_left_side"
node_options: {
[type.googleapis.com/mediapipe.SplitVectorCalculatorOptions] {
ranges: { begin: 1 end: 4 }
ranges: { begin: 7 end: 8 }
ranges: { begin: 9 end: 10 }
ranges: { begin: 11 end: 12 }
ranges: { begin: 13 end: 14 }
ranges: { begin: 15 end: 16 }
ranges: { begin: 17 end: 18 }
ranges: { begin: 19 end: 20 }
ranges: { begin: 21 end: 22 }
ranges: { begin: 23 end: 24 }
combine_outputs: true
}
}
}
# Take right pose landmarks.
node {
calculator: "SplitNormalizedLandmarkListCalculator"
input_stream: "pose_landmarks"
output_stream: "landmarks_right_side"
node_options: {
[type.googleapis.com/mediapipe.SplitVectorCalculatorOptions] {
ranges: { begin: 4 end: 7 }
ranges: { begin: 8 end: 9 }
ranges: { begin: 10 end: 11 }
ranges: { begin: 12 end: 13 }
ranges: { begin: 14 end: 15 }
ranges: { begin: 16 end: 17 }
ranges: { begin: 18 end: 19 }
ranges: { begin: 20 end: 21 }
ranges: { begin: 22 end: 23 }
ranges: { begin: 24 end: 25 }
combine_outputs: true
}
}
}
# Render pose joints as big white circles.
node {
calculator: "LandmarksToRenderDataCalculator"
input_stream: "NORM_LANDMARKS:visible_pose_landmarks"
input_stream: "RENDER_SCALE:render_scale"
output_stream: "RENDER_DATA:landmarks_background_joints_render_data"
node_options: {
[type.googleapis.com/mediapipe.LandmarksToRenderDataCalculatorOptions] {
landmark_color { r: 255 g: 255 b: 255 }
connection_color { r: 255 g: 255 b: 255 }
thickness: 5.0
visualize_landmark_depth: false
utilize_visibility: true
visibility_threshold: 0.1
}
}
}
# Render pose left side joints as orange circles (inside white ones).
node {
calculator: "LandmarksToRenderDataCalculator"
input_stream: "NORM_LANDMARKS:landmarks_left_side"
input_stream: "RENDER_SCALE:render_scale"
output_stream: "RENDER_DATA:landmarks_left_joints_render_data"
node_options: {
[type.googleapis.com/mediapipe.LandmarksToRenderDataCalculatorOptions] {
landmark_color { r: 255 g: 138 b: 0 }
connection_color { r: 255 g: 138 b: 0 }
thickness: 3.0
visualize_landmark_depth: false
utilize_visibility: true
visibility_threshold: 0.1
}
}
}
# Render pose right side joints as cyan circles (inside white ones).
node {
calculator: "LandmarksToRenderDataCalculator"
input_stream: "NORM_LANDMARKS:landmarks_right_side"
input_stream: "RENDER_SCALE:render_scale"
output_stream: "RENDER_DATA:landmarks_right_joints_render_data"
node_options: {
[type.googleapis.com/mediapipe.LandmarksToRenderDataCalculatorOptions] {
landmark_color { r: 0 g: 217 b: 231 }
connection_color { r: 0 g: 217 b: 231 }
thickness: 3.0
visualize_landmark_depth: false
utilize_visibility: true
visibility_threshold: 0.1
}
}
}
# Converts normalized rects to drawing primitives for annotation overlay.
node {
calculator: "RectToRenderDataCalculator"
input_stream: "NORM_RECT:roi"
output_stream: "RENDER_DATA:roi_render_data"
node_options: {
[type.googleapis.com/mediapipe.RectToRenderDataCalculatorOptions] {
filled: false
color { r: 255 g: 0 b: 0 }
thickness: 4.0
}
}
}
# Draws annotations and overlays them on top of the input images.
node {
calculator: "AnnotationOverlayCalculator"
input_stream: "IMAGE:input_image"
input_stream: "detection_render_data"
input_stream: "landmarks_render_data"
input_stream: "landmarks_background_joints_render_data"
input_stream: "landmarks_left_joints_render_data"
input_stream: "landmarks_right_joints_render_data"
input_stream: "roi_render_data"
output_stream: "IMAGE:output_image"
}
mediapipe/graphs/pose_tracking/subgraphs/upper_body_pose_renderer_gpu.pbtxt
+254
View File
@@ -0,0 +1,254 @@
# MediaPipe pose landmarks rendering subgraph.
type: "UpperBodyPoseRendererGpu"
# GPU image. (GpuBuffer)
input_stream: "IMAGE:input_image"
# Pose landmarks. (NormalizedLandmarkList)
input_stream: "LANDMARKS:pose_landmarks"
# Region of interest calculated based on landmarks. (NormalizedRect)
input_stream: "ROI:roi"
# Detected pose. (Detection)
input_stream: "DETECTION:detection"
# GPU image with rendered data. (GpuBuffer)
output_stream: "IMAGE:output_image"
node {
calculator: "ImagePropertiesCalculator"
input_stream: "IMAGE_GPU:input_image"
output_stream: "SIZE:image_size"
}
# Calculates rendering scale based on the pose roi.
node {
calculator: "RectToRenderScaleCalculator"
input_stream: "NORM_RECT:roi"
input_stream: "IMAGE_SIZE:image_size"
output_stream: "RENDER_SCALE:render_scale"
node_options: {
[type.googleapis.com/mediapipe.RectToRenderScaleCalculatorOptions] {
multiplier: 0.0012
}
}
}
# Converts detections to drawing primitives for annotation overlay.
node {
calculator: "DetectionsToRenderDataCalculator"
input_stream: "DETECTION:detection"
output_stream: "RENDER_DATA:detection_render_data"
node_options: {
[type.googleapis.com/mediapipe.DetectionsToRenderDataCalculatorOptions] {
thickness: 4.0
color { r: 0 g: 255 b: 0 }
}
}
}
node {
calculator: "SplitNormalizedLandmarkListCalculator"
input_stream: "pose_landmarks"
output_stream: "visible_pose_landmarks"
node_options: {
[type.googleapis.com/mediapipe.SplitVectorCalculatorOptions] {
ranges: { begin: 0 end: 25 }
}
}
}
# Converts landmarks to drawing primitives for annotation overlay.
node {
calculator: "LandmarksToRenderDataCalculator"
input_stream: "NORM_LANDMARKS:pose_landmarks"
input_stream: "RENDER_SCALE:render_scale"
output_stream: "RENDER_DATA:landmarks_render_data"
node_options: {
[type.googleapis.com/mediapipe.LandmarksToRenderDataCalculatorOptions] {
landmark_connections: 0
landmark_connections: 1
landmark_connections: 1
landmark_connections: 2
landmark_connections: 2
landmark_connections: 3
landmark_connections: 3
landmark_connections: 7
landmark_connections: 0
landmark_connections: 4
landmark_connections: 4
landmark_connections: 5
landmark_connections: 5
landmark_connections: 6
landmark_connections: 6
landmark_connections: 8
landmark_connections: 9
landmark_connections: 10
landmark_connections: 11
landmark_connections: 12
landmark_connections: 11
landmark_connections: 13
landmark_connections: 13
landmark_connections: 15
landmark_connections: 15
landmark_connections: 17
landmark_connections: 15
landmark_connections: 19
landmark_connections: 15
landmark_connections: 21
landmark_connections: 17
landmark_connections: 19
landmark_connections: 12
landmark_connections: 14
landmark_connections: 14
landmark_connections: 16
landmark_connections: 16
landmark_connections: 18
landmark_connections: 16
landmark_connections: 20
landmark_connections: 16
landmark_connections: 22
landmark_connections: 18
landmark_connections: 20
landmark_connections: 11
landmark_connections: 23
landmark_connections: 12
landmark_connections: 24
landmark_connections: 23
landmark_connections: 24
landmark_color { r: 255 g: 255 b: 255 }
connection_color { r: 255 g: 255 b: 255 }
thickness: 3.0
visualize_landmark_depth: false
utilize_visibility: true
visibility_threshold: 0.1
}
}
}
# Take left pose landmarks.
node {
calculator: "SplitNormalizedLandmarkListCalculator"
input_stream: "pose_landmarks"
output_stream: "landmarks_left_side"
node_options: {
[type.googleapis.com/mediapipe.SplitVectorCalculatorOptions] {
ranges: { begin: 1 end: 4 }
ranges: { begin: 7 end: 8 }
ranges: { begin: 9 end: 10 }
ranges: { begin: 11 end: 12 }
ranges: { begin: 13 end: 14 }
ranges: { begin: 15 end: 16 }
ranges: { begin: 17 end: 18 }
ranges: { begin: 19 end: 20 }
ranges: { begin: 21 end: 22 }
ranges: { begin: 23 end: 24 }
combine_outputs: true
}
}
}
# Take right pose landmarks.
node {
calculator: "SplitNormalizedLandmarkListCalculator"
input_stream: "pose_landmarks"
output_stream: "landmarks_right_side"
node_options: {
[type.googleapis.com/mediapipe.SplitVectorCalculatorOptions] {
ranges: { begin: 4 end: 7 }
ranges: { begin: 8 end: 9 }
ranges: { begin: 10 end: 11 }
ranges: { begin: 12 end: 13 }
ranges: { begin: 14 end: 15 }
ranges: { begin: 16 end: 17 }
ranges: { begin: 18 end: 19 }
ranges: { begin: 20 end: 21 }
ranges: { begin: 22 end: 23 }
ranges: { begin: 24 end: 25 }
combine_outputs: true
}
}
}
# Render pose joints as big white circles.
node {
calculator: "LandmarksToRenderDataCalculator"
input_stream: "NORM_LANDMARKS:visible_pose_landmarks"
input_stream: "RENDER_SCALE:render_scale"
output_stream: "RENDER_DATA:landmarks_background_joints_render_data"
node_options: {
[type.googleapis.com/mediapipe.LandmarksToRenderDataCalculatorOptions] {
landmark_color { r: 255 g: 255 b: 255 }
connection_color { r: 255 g: 255 b: 255 }
thickness: 5.0
visualize_landmark_depth: false
utilize_visibility: true
visibility_threshold: 0.1
}
}
}
# Render pose left side joints as orange circles (inside white ones).
node {
calculator: "LandmarksToRenderDataCalculator"
input_stream: "NORM_LANDMARKS:landmarks_left_side"
input_stream: "RENDER_SCALE:render_scale"
output_stream: "RENDER_DATA:landmarks_left_joints_render_data"
node_options: {
[type.googleapis.com/mediapipe.LandmarksToRenderDataCalculatorOptions] {
landmark_color { r: 255 g: 138 b: 0 }
connection_color { r: 255 g: 138 b: 0 }
thickness: 3.0
visualize_landmark_depth: false
utilize_visibility: true
visibility_threshold: 0.1
}
}
}
# Render pose right side joints as cyan circles (inside white ones).
node {
calculator: "LandmarksToRenderDataCalculator"
input_stream: "NORM_LANDMARKS:landmarks_right_side"
input_stream: "RENDER_SCALE:render_scale"
output_stream: "RENDER_DATA:landmarks_right_joints_render_data"
node_options: {
[type.googleapis.com/mediapipe.LandmarksToRenderDataCalculatorOptions] {
landmark_color { r: 0 g: 217 b: 231 }
connection_color { r: 0 g: 217 b: 231 }
thickness: 3.0
visualize_landmark_depth: false
utilize_visibility: true
visibility_threshold: 0.1
}
}
}
# Converts normalized rects to drawing primitives for annotation overlay.
node {
calculator: "RectToRenderDataCalculator"
input_stream: "NORM_RECT:roi"
output_stream: "RENDER_DATA:roi_render_data"
node_options: {
[type.googleapis.com/mediapipe.RectToRenderDataCalculatorOptions] {
filled: false
color { r: 255 g: 0 b: 0 }
thickness: 4.0
}
}
}
# Draws annotations and overlays them on top of the input images.
node {
calculator: "AnnotationOverlayCalculator"
input_stream: "IMAGE_GPU:input_image"
input_stream: "detection_render_data"
input_stream: "landmarks_render_data"
input_stream: "landmarks_background_joints_render_data"
input_stream: "landmarks_left_joints_render_data"
input_stream: "landmarks_right_joints_render_data"
input_stream: "roi_render_data"
output_stream: "IMAGE_GPU:output_image"
}
mediapipe/graphs/pose_tracking/upper_body_pose_tracking_cpu.pbtxt
+72
View File
@@ -0,0 +1,72 @@
# MediaPipe graph that performs upper-body pose tracking with TensorFlow Lite on CPU.
# CPU buffer. (ImageFrame)
input_stream: "input_video"
# Output image with rendered results. (ImageFrame)
output_stream: "output_video"
# Pose landmarks. (NormalizedLandmarkList)
output_stream: "pose_landmarks"
# Throttles the images flowing downstream for flow control. It passes through
# the very first incoming image unaltered, and waits for downstream nodes
# (calculators and subgraphs) in the graph to finish their tasks before it
# passes through another image. All images that come in while waiting are
# dropped, limiting the number of in-flight images in most part of the graph to
# 1. This prevents the downstream nodes from queuing up incoming images and data
# excessively, which leads to increased latency and memory usage, unwanted in
# real-time mobile applications. It also eliminates unnecessarily computation,
# e.g., the output produced by a node may get dropped downstream if the
# subsequent nodes are still busy processing previous inputs.
node {
calculator: "FlowLimiterCalculator"
input_stream: "input_video"
input_stream: "FINISHED:output_video"
input_stream_info: {
tag_index: "FINISHED"
back_edge: true
}
output_stream: "throttled_input_video"
}
# Subgraph that detects poses and corresponding landmarks.
node {
calculator: "PoseLandmarkUpperBodyCpu"
input_stream: "IMAGE:throttled_input_video"
output_stream: "LANDMARKS:pose_landmarks"
output_stream: "DETECTION:pose_detection"
output_stream: "ROI_FROM_LANDMARKS:roi_from_landmarks"
}
# Calculates size of the image.
node {
calculator: "ImagePropertiesCalculator"
input_stream: "IMAGE:throttled_input_video"
output_stream: "SIZE:image_size"
}
# Smoothes pose landmarks in order to reduce jitter.
node {
calculator: "LandmarksSmoothingCalculator"
input_stream: "NORM_LANDMARKS:pose_landmarks"
input_stream: "IMAGE_SIZE:image_size"
output_stream: "NORM_FILTERED_LANDMARKS:pose_landmarks_smoothed"
node_options: {
[type.googleapis.com/mediapipe.LandmarksSmoothingCalculatorOptions] {
velocity_filter: {
window_size: 5
velocity_scale: 10.0
}
}
}
}
# Subgraph that renders pose-landmark annotation onto the input image.
node {
calculator: "UpperBodyPoseRendererCpu"
input_stream: "IMAGE:throttled_input_video"
input_stream: "LANDMARKS:pose_landmarks_smoothed"
input_stream: "ROI:roi_from_landmarks"
input_stream: "DETECTION:pose_detection"
output_stream: "IMAGE:output_video"
}
mediapipe/graphs/pose_tracking/upper_body_pose_tracking_gpu.pbtxt
+72
View File
@@ -0,0 +1,72 @@
# MediaPipe graph that performs upper-body pose tracking with TensorFlow Lite on GPU.
# GPU buffer. (GpuBuffer)
input_stream: "input_video"
# Output image with rendered results. (GpuBuffer)
output_stream: "output_video"
# Pose landmarks. (NormalizedLandmarkList)
output_stream: "pose_landmarks"
# Throttles the images flowing downstream for flow control. It passes through
# the very first incoming image unaltered, and waits for downstream nodes
# (calculators and subgraphs) in the graph to finish their tasks before it
# passes through another image. All images that come in while waiting are
# dropped, limiting the number of in-flight images in most part of the graph to
# 1. This prevents the downstream nodes from queuing up incoming images and data
# excessively, which leads to increased latency and memory usage, unwanted in
# real-time mobile applications. It also eliminates unnecessarily computation,
# e.g., the output produced by a node may get dropped downstream if the
# subsequent nodes are still busy processing previous inputs.
node {
calculator: "FlowLimiterCalculator"
input_stream: "input_video"
input_stream: "FINISHED:output_video"
input_stream_info: {
tag_index: "FINISHED"
back_edge: true
}
output_stream: "throttled_input_video"
}
# Subgraph that detects poses and corresponding landmarks.
node {
calculator: "PoseLandmarkUpperBodyGpu"
input_stream: "IMAGE:throttled_input_video"
output_stream: "LANDMARKS:pose_landmarks"
output_stream: "DETECTION:pose_detection"
output_stream: "ROI_FROM_LANDMARKS:roi_from_landmarks"
}
# Calculates size of the image.
node {
calculator: "ImagePropertiesCalculator"
input_stream: "IMAGE_GPU:throttled_input_video"
output_stream: "SIZE:image_size"
}
# Smoothes pose landmarks in order to reduce jitter.
node {
calculator: "LandmarksSmoothingCalculator"
input_stream: "NORM_LANDMARKS:pose_landmarks"
input_stream: "IMAGE_SIZE:image_size"
output_stream: "NORM_FILTERED_LANDMARKS:pose_landmarks_smoothed"
node_options: {
[type.googleapis.com/mediapipe.LandmarksSmoothingCalculatorOptions] {
velocity_filter: {
window_size: 5
velocity_scale: 10.0
}
}
}
}
# Subgraph that renders pose-landmark annotation onto the input image.
node {
calculator: "UpperBodyPoseRendererGpu"
input_stream: "IMAGE:throttled_input_video"
input_stream: "LANDMARKS:pose_landmarks_smoothed"
input_stream: "ROI:roi_from_landmarks"
input_stream: "DETECTION:pose_detection"
output_stream: "IMAGE:output_video"
}