Project import generated by Copybara.
GitOrigin-RevId: 4cee4a2c2317fb190680c17e31ebbb03bb73b71c
This commit is contained in:
MediaPipe Team
@@ -0,0 +1,43 @@
|
||||
# Copyright 2020 The MediaPipe Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
load(
|
||||
"//mediapipe/framework/tool:mediapipe_graph.bzl",
|
||||
"mediapipe_binary_graph",
|
||||
)
|
||||
|
||||
licenses(["notice"])
|
||||
|
||||
package(default_visibility = ["//visibility:public"])
|
||||
|
||||
cc_library(
|
||||
name = "face_effect_gpu_deps",
|
||||
deps = [
|
||||
"//mediapipe/calculators/core:flow_limiter_calculator",
|
||||
"//mediapipe/calculators/core:gate_calculator",
|
||||
"//mediapipe/calculators/core:immediate_mux_calculator",
|
||||
"//mediapipe/calculators/image:image_properties_calculator",
|
||||
"//mediapipe/graphs/face_effect/subgraphs:single_face_smooth_landmark_gpu",
|
||||
"//mediapipe/modules/face_geometry",
|
||||
"//mediapipe/modules/face_geometry:effect_renderer_calculator",
|
||||
"//mediapipe/modules/face_geometry:env_generator_calculator",
|
||||
],
|
||||
)
|
||||
|
||||
mediapipe_binary_graph(
|
||||
name = "face_effect_gpu_binary_graph",
|
||||
graph = "face_effect_gpu.pbtxt",
|
||||
output_name = "face_effect_gpu.binarypb",
|
||||
deps = [":face_effect_gpu_deps"],
|
||||
)
|
||||
@@ -0,0 +1,36 @@
|
||||
# Copyright 2020 The MediaPipe Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
load("//mediapipe/framework:encode_binary_proto.bzl", "encode_binary_proto")
|
||||
|
||||
licenses(["notice"])
|
||||
|
||||
package(default_visibility = ["//visibility:public"])
|
||||
|
||||
encode_binary_proto(
|
||||
name = "glasses",
|
||||
input = "glasses.pbtxt",
|
||||
message_type = "mediapipe.face_geometry.Mesh3d",
|
||||
output = "glasses.binarypb",
|
||||
deps = [
|
||||
"//mediapipe/modules/face_geometry/protos:mesh_3d_proto",
|
||||
],
|
||||
)
|
||||
|
||||
# `.pngblob` is used instead of `.png` to prevent iOS build from preprocessing the image.
|
||||
# OpenCV is unable to read a PNG file preprocessed by the iOS build.
|
||||
exports_files([
|
||||
"facepaint.pngblob",
|
||||
"glasses.pngblob",
|
||||
])
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 593 KiB |
File diff suppressed because it is too large
Load Diff
Binary file not shown.
|
After Width: | Height: | Size: 293 KiB |
@@ -0,0 +1,145 @@
|
||||
# MediaPipe graph that applies a face effect to the input video stream.
|
||||
|
||||
# GPU buffer. (GpuBuffer)
|
||||
input_stream: "input_video"
|
||||
|
||||
# Boolean flag, which indicates whether the Facepaint effect is selected. (bool)
|
||||
#
|
||||
# If `true`, the Facepaint effect will be rendered.
|
||||
# If `false`, the Glasses effect will be rendered.
|
||||
input_stream: "is_facepaint_effect_selected"
|
||||
|
||||
# Output image with rendered results. (GpuBuffer)
|
||||
output_stream: "output_video"
|
||||
|
||||
# A list of geometry data for a single detected face.
|
||||
#
|
||||
# NOTE: there will not be an output packet in this stream for this particular
|
||||
# timestamp if none of faces detected.
|
||||
#
|
||||
# (std::vector<face_geometry::FaceGeometry>)
|
||||
output_stream: "multi_face_geometry"
|
||||
|
||||
# Throttles the images flowing downstream for flow control. It passes through
|
||||
# the very first incoming image unaltered, and waits for downstream nodes
|
||||
# (calculators and subgraphs) in the graph to finish their tasks before it
|
||||
# passes through another image. All images that come in while waiting are
|
||||
# dropped, limiting the number of in-flight images in most part of the graph to
|
||||
# 1. This prevents the downstream nodes from queuing up incoming images and data
|
||||
# excessively, which leads to increased latency and memory usage, unwanted in
|
||||
# real-time mobile applications. It also eliminates unnecessarily computation,
|
||||
# e.g., the output produced by a node may get dropped downstream if the
|
||||
# subsequent nodes are still busy processing previous inputs.
|
||||
node {
|
||||
calculator: "FlowLimiterCalculator"
|
||||
input_stream: "input_video"
|
||||
input_stream: "FINISHED:output_video"
|
||||
input_stream_info: {
|
||||
tag_index: "FINISHED"
|
||||
back_edge: true
|
||||
}
|
||||
output_stream: "throttled_input_video"
|
||||
}
|
||||
|
||||
# Generates an environment that describes the current virtual scene.
|
||||
node {
|
||||
calculator: "FaceGeometryEnvGeneratorCalculator"
|
||||
output_side_packet: "ENVIRONMENT:environment"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.FaceGeometryEnvGeneratorCalculatorOptions] {
|
||||
environment: {
|
||||
origin_point_location: TOP_LEFT_CORNER
|
||||
perspective_camera: {
|
||||
vertical_fov_degrees: 63.0 # 63 degrees
|
||||
near: 1.0 # 1cm
|
||||
far: 10000.0 # 100m
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Subgraph that detects a single face and corresponding landmarks. The landmarks
|
||||
# are also "smoothed" to achieve better visual results.
|
||||
node {
|
||||
calculator: "SingleFaceSmoothLandmarkGpu"
|
||||
input_stream: "IMAGE:throttled_input_video"
|
||||
output_stream: "LANDMARKS:multi_face_landmarks"
|
||||
}
|
||||
|
||||
# Extracts the throttled input video frame dimensions as a separate packet.
|
||||
node {
|
||||
calculator: "ImagePropertiesCalculator"
|
||||
input_stream: "IMAGE_GPU:throttled_input_video"
|
||||
output_stream: "SIZE:input_video_size"
|
||||
}
|
||||
|
||||
# Subgraph that computes face geometry from landmarks for a single face.
|
||||
node {
|
||||
calculator: "FaceGeometry"
|
||||
input_stream: "MULTI_FACE_LANDMARKS:multi_face_landmarks"
|
||||
input_stream: "IMAGE_SIZE:input_video_size"
|
||||
input_side_packet: "ENVIRONMENT:environment"
|
||||
output_stream: "MULTI_FACE_GEOMETRY:multi_face_geometry"
|
||||
}
|
||||
|
||||
# Decides whether to render the Facepaint effect based on the
|
||||
# `is_facepaint_effect_selected` flag value.
|
||||
node {
|
||||
calculator: "GateCalculator"
|
||||
input_stream: "throttled_input_video"
|
||||
input_stream: "multi_face_geometry"
|
||||
input_stream: "ALLOW:is_facepaint_effect_selected"
|
||||
output_stream: "facepaint_effect_throttled_input_video"
|
||||
output_stream: "facepaint_effect_multi_face_geometry"
|
||||
}
|
||||
|
||||
# Renders the Facepaint effect.
|
||||
node {
|
||||
calculator: "FaceGeometryEffectRendererCalculator"
|
||||
input_side_packet: "ENVIRONMENT:environment"
|
||||
input_stream: "IMAGE_GPU:facepaint_effect_throttled_input_video"
|
||||
input_stream: "MULTI_FACE_GEOMETRY:facepaint_effect_multi_face_geometry"
|
||||
output_stream: "IMAGE_GPU:facepaint_effect_output_video"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.FaceGeometryEffectRendererCalculatorOptions] {
|
||||
effect_texture_path: "mediapipe/graphs/face_effect/data/facepaint.pngblob"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Decides whether to render the Glasses effect based on the
|
||||
# `is_facepaint_effect_selected` flag value.
|
||||
node {
|
||||
calculator: "GateCalculator"
|
||||
input_stream: "throttled_input_video"
|
||||
input_stream: "multi_face_geometry"
|
||||
input_stream: "DISALLOW:is_facepaint_effect_selected"
|
||||
output_stream: "glasses_effect_throttled_input_video"
|
||||
output_stream: "glasses_effect_multi_face_geometry"
|
||||
}
|
||||
|
||||
# Renders the Glasses effect.
|
||||
node {
|
||||
calculator: "FaceGeometryEffectRendererCalculator"
|
||||
input_side_packet: "ENVIRONMENT:environment"
|
||||
input_stream: "IMAGE_GPU:glasses_effect_throttled_input_video"
|
||||
input_stream: "MULTI_FACE_GEOMETRY:glasses_effect_multi_face_geometry"
|
||||
output_stream: "IMAGE_GPU:glasses_effect_output_video"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.FaceGeometryEffectRendererCalculatorOptions] {
|
||||
effect_texture_path: "mediapipe/graphs/face_effect/data/glasses.pngblob"
|
||||
effect_mesh_3d_path: "mediapipe/graphs/face_effect/data/glasses.binarypb"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Decides which of the Facepaint or the Glasses rendered results should be sent
|
||||
# as the output GPU frame.
|
||||
node {
|
||||
calculator: "ImmediateMuxCalculator"
|
||||
input_stream: "facepaint_effect_output_video"
|
||||
input_stream: "glasses_effect_output_video"
|
||||
output_stream: "output_video"
|
||||
}
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
# Copyright 2020 The MediaPipe Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
load(
|
||||
"//mediapipe/framework/tool:mediapipe_graph.bzl",
|
||||
"mediapipe_simple_subgraph",
|
||||
)
|
||||
|
||||
licenses(["notice"])
|
||||
|
||||
package(default_visibility = ["//visibility:public"])
|
||||
|
||||
mediapipe_simple_subgraph(
|
||||
name = "single_face_smooth_landmark_gpu",
|
||||
graph = "single_face_smooth_landmark_gpu.pbtxt",
|
||||
register_as = "SingleFaceSmoothLandmarkGpu",
|
||||
deps = [
|
||||
"//mediapipe/calculators/core:concatenate_vector_calculator",
|
||||
"//mediapipe/calculators/core:constant_side_packet_calculator",
|
||||
"//mediapipe/calculators/core:split_vector_calculator",
|
||||
"//mediapipe/calculators/image:image_properties_calculator",
|
||||
"//mediapipe/calculators/util:landmarks_smoothing_calculator",
|
||||
"//mediapipe/modules/face_landmark:face_landmark_front_gpu",
|
||||
],
|
||||
)
|
||||
@@ -0,0 +1,84 @@
|
||||
# MediaPipe subgraph that detects a single face and corresponding landmarks on
|
||||
# a input GPU image. The landmarks are also "smoothed" to achieve better visual
|
||||
# results.
|
||||
|
||||
type: "SingleFaceSmoothLandmarkGpu"
|
||||
|
||||
# GPU image. (GpuBuffer)
|
||||
input_stream: "IMAGE:input_image"
|
||||
|
||||
# Collection of detected/predicted faces, each represented as a list of face
|
||||
# landmarks. However, the size of this collection is always 1 because of the
|
||||
# single-face use in this graph. The decision to wrap the landmark list into a
|
||||
# collection was made to simplify passing the result into the `FaceGeometry`
|
||||
# subgraph. (std::vector<NormalizedLandmarkList>)
|
||||
#
|
||||
# NOTE: there will not be an output packet in the LANDMARKS stream for this
|
||||
# particular timestamp if none of faces detected. However, the MediaPipe
|
||||
# framework will internally inform the downstream calculators of the absence of
|
||||
# this packet so that they don't wait for it unnecessarily.
|
||||
output_stream: "LANDMARKS:multi_face_smooth_landmarks"
|
||||
|
||||
# Creates a packet to inform the `FaceLandmarkFrontGpu` subgraph to detect at
|
||||
# most 1 face.
|
||||
node {
|
||||
calculator: "ConstantSidePacketCalculator"
|
||||
output_side_packet: "PACKET:num_faces"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.ConstantSidePacketCalculatorOptions]: {
|
||||
packet { int_value: 1 }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Subgraph that detects faces and corresponding landmarks.
|
||||
node {
|
||||
calculator: "FaceLandmarkFrontGpu"
|
||||
input_stream: "IMAGE:input_image"
|
||||
input_side_packet: "NUM_FACES:num_faces"
|
||||
output_stream: "LANDMARKS:multi_face_landmarks"
|
||||
}
|
||||
|
||||
# Extracts the detected face landmark list from a collection.
|
||||
node {
|
||||
calculator: "SplitNormalizedLandmarkListVectorCalculator"
|
||||
input_stream: "multi_face_landmarks"
|
||||
output_stream: "face_landmarks"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.SplitVectorCalculatorOptions] {
|
||||
ranges: { begin: 0 end: 1 }
|
||||
element_only: true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Extracts the input image frame dimensions as a separate packet.
|
||||
node {
|
||||
calculator: "ImagePropertiesCalculator"
|
||||
input_stream: "IMAGE_GPU:input_image"
|
||||
output_stream: "SIZE:input_image_size"
|
||||
}
|
||||
|
||||
# Applies smoothing to the single face landmarks.
|
||||
node {
|
||||
calculator: "LandmarksSmoothingCalculator"
|
||||
input_stream: "NORM_LANDMARKS:face_landmarks"
|
||||
input_stream: "IMAGE_SIZE:input_image_size"
|
||||
output_stream: "NORM_FILTERED_LANDMARKS:face_smooth_landmarks"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.LandmarksSmoothingCalculatorOptions] {
|
||||
velocity_filter: {
|
||||
window_size: 5
|
||||
velocity_scale: 20.0
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Puts the single face smooth landmarks back into a collection to simplify
|
||||
# passing the result into the `FaceGeometry` subgraph.
|
||||
node {
|
||||
calculator: "ConcatenateLandmarListVectorCalculator"
|
||||
input_stream: "face_smooth_landmarks"
|
||||
output_stream: "multi_face_smooth_landmarks"
|
||||
}
|
||||
@@ -16,7 +16,7 @@ load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library"
|
||||
|
||||
licenses(["notice"])
|
||||
|
||||
package(default_visibility = ["//visibility:private"])
|
||||
package(default_visibility = ["//visibility:public"])
|
||||
|
||||
proto_library(
|
||||
name = "object_proto",
|
||||
|
||||
@@ -26,7 +26,7 @@ cc_library(
|
||||
deps = [
|
||||
"//mediapipe/calculators/core:flow_limiter_calculator",
|
||||
"//mediapipe/calculators/image:image_properties_calculator",
|
||||
"//mediapipe/graphs/pose_tracking/calculators:landmarks_smoothing_calculator",
|
||||
"//mediapipe/calculators/util:landmarks_smoothing_calculator",
|
||||
"//mediapipe/graphs/pose_tracking/subgraphs:upper_body_pose_renderer_gpu",
|
||||
"//mediapipe/modules/pose_landmark:pose_landmark_upper_body_gpu",
|
||||
],
|
||||
@@ -44,7 +44,7 @@ cc_library(
|
||||
deps = [
|
||||
"//mediapipe/calculators/core:flow_limiter_calculator",
|
||||
"//mediapipe/calculators/image:image_properties_calculator",
|
||||
"//mediapipe/graphs/pose_tracking/calculators:landmarks_smoothing_calculator",
|
||||
"//mediapipe/calculators/util:landmarks_smoothing_calculator",
|
||||
"//mediapipe/graphs/pose_tracking/subgraphs:upper_body_pose_renderer_cpu",
|
||||
"//mediapipe/modules/pose_landmark:pose_landmark_upper_body_cpu",
|
||||
],
|
||||
|
||||
@@ -1,85 +0,0 @@
|
||||
# Copyright 2020 The MediaPipe Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library")
|
||||
|
||||
licenses(["notice"])
|
||||
|
||||
package(default_visibility = ["//visibility:public"])
|
||||
|
||||
cc_library(
|
||||
name = "low_pass_filter",
|
||||
srcs = ["low_pass_filter.cc"],
|
||||
hdrs = ["low_pass_filter.h"],
|
||||
deps = [
|
||||
"//mediapipe/framework/port:logging",
|
||||
"@com_google_absl//absl/memory",
|
||||
],
|
||||
)
|
||||
|
||||
cc_test(
|
||||
name = "low_pass_filter_test",
|
||||
srcs = ["low_pass_filter_test.cc"],
|
||||
deps = [
|
||||
":low_pass_filter",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "relative_velocity_filter",
|
||||
srcs = ["relative_velocity_filter.cc"],
|
||||
hdrs = ["relative_velocity_filter.h"],
|
||||
deps = [
|
||||
":low_pass_filter",
|
||||
"//mediapipe/framework/port:logging",
|
||||
"@com_google_absl//absl/memory",
|
||||
"@com_google_absl//absl/time",
|
||||
],
|
||||
)
|
||||
|
||||
cc_test(
|
||||
name = "relative_velocity_filter_test",
|
||||
srcs = ["relative_velocity_filter_test.cc"],
|
||||
deps = [
|
||||
":relative_velocity_filter",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"@com_google_absl//absl/time",
|
||||
],
|
||||
)
|
||||
|
||||
mediapipe_proto_library(
|
||||
name = "landmarks_smoothing_calculator_proto",
|
||||
srcs = ["landmarks_smoothing_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_proto",
|
||||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "landmarks_smoothing_calculator",
|
||||
srcs = ["landmarks_smoothing_calculator.cc"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
":landmarks_smoothing_calculator_cc_proto",
|
||||
":relative_velocity_filter",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:timestamp",
|
||||
"//mediapipe/framework/formats:landmark_cc_proto",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"@com_google_absl//absl/algorithm:container",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
@@ -1,273 +0,0 @@
|
||||
// Copyright 2020 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "absl/algorithm/container.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/formats/landmark.pb.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/framework/timestamp.h"
|
||||
#include "mediapipe/graphs/pose_tracking/calculators/landmarks_smoothing_calculator.pb.h"
|
||||
#include "mediapipe/graphs/pose_tracking/calculators/relative_velocity_filter.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
namespace {
|
||||
|
||||
constexpr char kNormalizedLandmarksTag[] = "NORM_LANDMARKS";
|
||||
constexpr char kImageSizeTag[] = "IMAGE_SIZE";
|
||||
constexpr char kNormalizedFilteredLandmarksTag[] = "NORM_FILTERED_LANDMARKS";
|
||||
|
||||
using ::mediapipe::RelativeVelocityFilter;
|
||||
|
||||
// Estimate object scale to use its inverse value as velocity scale for
|
||||
// RelativeVelocityFilter. If value will be too small (less than
|
||||
// `options_.min_allowed_object_scale`) smoothing will be disabled and
|
||||
// landmarks will be returned as is.
|
||||
// Object scale is calculated as average between bounding box width and height
|
||||
// with sides parallel to axis.
|
||||
float GetObjectScale(const NormalizedLandmarkList& landmarks, int image_width,
|
||||
int image_height) {
|
||||
const auto& [lm_min_x, lm_max_x] = absl::c_minmax_element(
|
||||
landmarks.landmark(),
|
||||
[](const auto& a, const auto& b) { return a.x() < b.x(); });
|
||||
const float x_min = lm_min_x->x();
|
||||
const float x_max = lm_max_x->x();
|
||||
|
||||
const auto& [lm_min_y, lm_max_y] = absl::c_minmax_element(
|
||||
landmarks.landmark(),
|
||||
[](const auto& a, const auto& b) { return a.y() < b.y(); });
|
||||
const float y_min = lm_min_y->y();
|
||||
const float y_max = lm_max_y->y();
|
||||
|
||||
const float object_width = (x_max - x_min) * image_width;
|
||||
const float object_height = (y_max - y_min) * image_height;
|
||||
|
||||
return (object_width + object_height) / 2.0f;
|
||||
}
|
||||
|
||||
// Abstract class for various landmarks filters.
|
||||
class LandmarksFilter {
|
||||
public:
|
||||
virtual ~LandmarksFilter() = default;
|
||||
|
||||
virtual ::mediapipe::Status Reset() { return ::mediapipe::OkStatus(); }
|
||||
|
||||
virtual ::mediapipe::Status Apply(const NormalizedLandmarkList& in_landmarks,
|
||||
const std::pair<int, int>& image_size,
|
||||
const absl::Duration& timestamp,
|
||||
NormalizedLandmarkList* out_landmarks) = 0;
|
||||
};
|
||||
|
||||
// Returns landmarks as is without smoothing.
|
||||
class NoFilter : public LandmarksFilter {
|
||||
public:
|
||||
::mediapipe::Status Apply(const NormalizedLandmarkList& in_landmarks,
|
||||
const std::pair<int, int>& image_size,
|
||||
const absl::Duration& timestamp,
|
||||
NormalizedLandmarkList* out_landmarks) override {
|
||||
*out_landmarks = in_landmarks;
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
};
|
||||
|
||||
// Please check RelativeVelocityFilter documentation for details.
|
||||
class VelocityFilter : public LandmarksFilter {
|
||||
public:
|
||||
VelocityFilter(int window_size, float velocity_scale,
|
||||
float min_allowed_object_scale)
|
||||
: window_size_(window_size),
|
||||
velocity_scale_(velocity_scale),
|
||||
min_allowed_object_scale_(min_allowed_object_scale) {}
|
||||
|
||||
::mediapipe::Status Reset() override {
|
||||
x_filters_.clear();
|
||||
y_filters_.clear();
|
||||
z_filters_.clear();
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status Apply(const NormalizedLandmarkList& in_landmarks,
|
||||
const std::pair<int, int>& image_size,
|
||||
const absl::Duration& timestamp,
|
||||
NormalizedLandmarkList* out_landmarks) override {
|
||||
// Get image size.
|
||||
int image_width;
|
||||
int image_height;
|
||||
std::tie(image_width, image_height) = image_size;
|
||||
|
||||
// Get value scale as inverse value of the object scale.
|
||||
// If value is too small smoothing will be disabled and landmarks will be
|
||||
// returned as is.
|
||||
const float object_scale =
|
||||
GetObjectScale(in_landmarks, image_width, image_height);
|
||||
if (object_scale < min_allowed_object_scale_) {
|
||||
*out_landmarks = in_landmarks;
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
const float value_scale = 1.0f / object_scale;
|
||||
|
||||
// Initialize filters once.
|
||||
MP_RETURN_IF_ERROR(InitializeFiltersIfEmpty(in_landmarks.landmark_size()));
|
||||
|
||||
// Filter landmarks. Every axis of every landmark is filtered separately.
|
||||
for (int i = 0; i < in_landmarks.landmark_size(); ++i) {
|
||||
const NormalizedLandmark& in_landmark = in_landmarks.landmark(i);
|
||||
|
||||
NormalizedLandmark* out_landmark = out_landmarks->add_landmark();
|
||||
out_landmark->set_x(x_filters_[i].Apply(timestamp, value_scale,
|
||||
in_landmark.x() * image_width) /
|
||||
image_width);
|
||||
out_landmark->set_y(y_filters_[i].Apply(timestamp, value_scale,
|
||||
in_landmark.y() * image_height) /
|
||||
image_height);
|
||||
// Scale Z the save was as X (using image width).
|
||||
out_landmark->set_z(z_filters_[i].Apply(timestamp, value_scale,
|
||||
in_landmark.z() * image_width) /
|
||||
image_width);
|
||||
// Keep visibility as is.
|
||||
out_landmark->set_visibility(in_landmark.visibility());
|
||||
}
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
private:
|
||||
// Initializes filters for the first time or after Reset. If initialized then
|
||||
// check the size.
|
||||
::mediapipe::Status InitializeFiltersIfEmpty(const int n_landmarks) {
|
||||
if (!x_filters_.empty()) {
|
||||
RET_CHECK_EQ(x_filters_.size(), n_landmarks);
|
||||
RET_CHECK_EQ(y_filters_.size(), n_landmarks);
|
||||
RET_CHECK_EQ(z_filters_.size(), n_landmarks);
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
x_filters_.resize(n_landmarks,
|
||||
RelativeVelocityFilter(window_size_, velocity_scale_));
|
||||
y_filters_.resize(n_landmarks,
|
||||
RelativeVelocityFilter(window_size_, velocity_scale_));
|
||||
z_filters_.resize(n_landmarks,
|
||||
RelativeVelocityFilter(window_size_, velocity_scale_));
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
int window_size_;
|
||||
float velocity_scale_;
|
||||
float min_allowed_object_scale_;
|
||||
|
||||
std::vector<RelativeVelocityFilter> x_filters_;
|
||||
std::vector<RelativeVelocityFilter> y_filters_;
|
||||
std::vector<RelativeVelocityFilter> z_filters_;
|
||||
};
|
||||
|
||||
} // namespace
|
||||
|
||||
// A calculator to smooth landmarks over time.
|
||||
//
|
||||
// Inputs:
|
||||
// NORM_LANDMARKS: A NormalizedLandmarkList of landmarks you want to smooth.
|
||||
// IMAGE_SIZE: A std::pair<int, int> represention of image width and height.
|
||||
// Required to perform all computations in absolute coordinates to avoid any
|
||||
// influence of normalized values.
|
||||
//
|
||||
// Outputs:
|
||||
// NORM_FILTERED_LANDMARKS: A NormalizedLandmarkList of smoothed landmarks.
|
||||
//
|
||||
// Example config:
|
||||
// node {
|
||||
// calculator: "LandmarksSmoothingCalculator"
|
||||
// input_stream: "NORM_LANDMARKS:pose_landmarks"
|
||||
// input_stream: "IMAGE_SIZE:image_size"
|
||||
// output_stream: "NORM_FILTERED_LANDMARKS:pose_landmarks_filtered"
|
||||
// node_options: {
|
||||
// [type.googleapis.com/mediapipe.LandmarksSmoothingCalculatorOptions] {
|
||||
// velocity_filter: {
|
||||
// window_size: 5
|
||||
// velocity_scale: 10.0
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//
|
||||
class LandmarksSmoothingCalculator : public CalculatorBase {
|
||||
public:
|
||||
static ::mediapipe::Status GetContract(CalculatorContract* cc);
|
||||
::mediapipe::Status Open(CalculatorContext* cc) override;
|
||||
::mediapipe::Status Process(CalculatorContext* cc) override;
|
||||
|
||||
private:
|
||||
LandmarksFilter* landmarks_filter_;
|
||||
};
|
||||
REGISTER_CALCULATOR(LandmarksSmoothingCalculator);
|
||||
|
||||
::mediapipe::Status LandmarksSmoothingCalculator::GetContract(
|
||||
CalculatorContract* cc) {
|
||||
cc->Inputs().Tag(kNormalizedLandmarksTag).Set<NormalizedLandmarkList>();
|
||||
cc->Inputs().Tag(kImageSizeTag).Set<std::pair<int, int>>();
|
||||
cc->Outputs()
|
||||
.Tag(kNormalizedFilteredLandmarksTag)
|
||||
.Set<NormalizedLandmarkList>();
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status LandmarksSmoothingCalculator::Open(CalculatorContext* cc) {
|
||||
cc->SetOffset(TimestampDiff(0));
|
||||
|
||||
// Pick landmarks filter.
|
||||
const auto& options = cc->Options<LandmarksSmoothingCalculatorOptions>();
|
||||
if (options.has_no_filter()) {
|
||||
landmarks_filter_ = new NoFilter();
|
||||
} else if (options.has_velocity_filter()) {
|
||||
landmarks_filter_ = new VelocityFilter(
|
||||
options.velocity_filter().window_size(),
|
||||
options.velocity_filter().velocity_scale(),
|
||||
options.velocity_filter().min_allowed_object_scale());
|
||||
} else {
|
||||
RET_CHECK_FAIL()
|
||||
<< "Landmarks filter is either not specified or not supported";
|
||||
}
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status LandmarksSmoothingCalculator::Process(
|
||||
CalculatorContext* cc) {
|
||||
// Check that landmarks are not empty and reset the filter if so.
|
||||
// Don't emit an empty packet for this timestamp.
|
||||
if (cc->Inputs().Tag(kNormalizedLandmarksTag).IsEmpty()) {
|
||||
MP_RETURN_IF_ERROR(landmarks_filter_->Reset());
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
const auto& in_landmarks =
|
||||
cc->Inputs().Tag(kNormalizedLandmarksTag).Get<NormalizedLandmarkList>();
|
||||
const auto& image_size =
|
||||
cc->Inputs().Tag(kImageSizeTag).Get<std::pair<int, int>>();
|
||||
const auto& timestamp =
|
||||
absl::Microseconds(cc->InputTimestamp().Microseconds());
|
||||
|
||||
auto out_landmarks = absl::make_unique<NormalizedLandmarkList>();
|
||||
MP_RETURN_IF_ERROR(landmarks_filter_->Apply(in_landmarks, image_size,
|
||||
timestamp, out_landmarks.get()));
|
||||
|
||||
cc->Outputs()
|
||||
.Tag(kNormalizedFilteredLandmarksTag)
|
||||
.Add(out_landmarks.release(), cc->InputTimestamp());
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
} // namespace mediapipe
|
||||
@@ -1,48 +0,0 @@
|
||||
// Copyright 2020 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
syntax = "proto2";
|
||||
|
||||
package mediapipe;
|
||||
|
||||
import "mediapipe/framework/calculator.proto";
|
||||
|
||||
message LandmarksSmoothingCalculatorOptions {
|
||||
extend CalculatorOptions {
|
||||
optional LandmarksSmoothingCalculatorOptions ext = 325671429;
|
||||
}
|
||||
|
||||
// Default behaviour and fast way to disable smoothing.
|
||||
message NoFilter {}
|
||||
|
||||
message VelocityFilter {
|
||||
// Number of value changes to keep over time.
|
||||
// Higher value adds to lag and to stability.
|
||||
optional int32 window_size = 1 [default = 5];
|
||||
|
||||
// Scale to apply to the velocity calculated over the given window. With
|
||||
// higher velocity `low pass filter` weights new values higher.
|
||||
// Lower value adds to lag and to stability.
|
||||
optional float velocity_scale = 2 [default = 10.0];
|
||||
|
||||
// If calculated object scale is less than given value smoothing will be
|
||||
// disabled and landmarks will be returned as is.
|
||||
optional float min_allowed_object_scale = 3 [default = 1e-6];
|
||||
}
|
||||
|
||||
oneof filter_options {
|
||||
NoFilter no_filter = 1;
|
||||
VelocityFilter velocity_filter = 2;
|
||||
}
|
||||
}
|
||||
@@ -1,58 +0,0 @@
|
||||
// Copyright 2020 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "mediapipe/graphs/pose_tracking/calculators/low_pass_filter.h"
|
||||
|
||||
#include "absl/memory/memory.h"
|
||||
#include "mediapipe/framework/port/logging.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
LowPassFilter::LowPassFilter(float alpha) : initialized_{false} {
|
||||
SetAlpha(alpha);
|
||||
}
|
||||
|
||||
float LowPassFilter::Apply(float value) {
|
||||
float result;
|
||||
if (initialized_) {
|
||||
result = alpha_ * value + (1.0 - alpha_) * stored_value_;
|
||||
} else {
|
||||
result = value;
|
||||
initialized_ = true;
|
||||
}
|
||||
raw_value_ = value;
|
||||
stored_value_ = result;
|
||||
return result;
|
||||
}
|
||||
|
||||
float LowPassFilter::ApplyWithAlpha(float value, float alpha) {
|
||||
SetAlpha(alpha);
|
||||
return Apply(value);
|
||||
}
|
||||
|
||||
bool LowPassFilter::HasLastRawValue() { return initialized_; }
|
||||
|
||||
float LowPassFilter::LastRawValue() { return raw_value_; }
|
||||
|
||||
float LowPassFilter::LastValue() { return stored_value_; }
|
||||
|
||||
void LowPassFilter::SetAlpha(float alpha) {
|
||||
if (alpha < 0.0f || alpha > 1.0f) {
|
||||
LOG(ERROR) << "alpha: " << alpha << " should be in [0.0, 1.0] range";
|
||||
return;
|
||||
}
|
||||
alpha_ = alpha;
|
||||
}
|
||||
|
||||
} // namespace mediapipe
|
||||
@@ -1,47 +0,0 @@
|
||||
// Copyright 2020 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef MEDIAPIPE_GRAPHS_POSE_TRACKING_CALCULATORS_LOW_PASS_FILTER_H_
|
||||
#define MEDIAPIPE_GRAPHS_POSE_TRACKING_CALCULATORS_LOW_PASS_FILTER_H_
|
||||
|
||||
#include <memory>
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
class LowPassFilter {
|
||||
public:
|
||||
explicit LowPassFilter(float alpha);
|
||||
|
||||
float Apply(float value);
|
||||
|
||||
float ApplyWithAlpha(float value, float alpha);
|
||||
|
||||
bool HasLastRawValue();
|
||||
|
||||
float LastRawValue();
|
||||
|
||||
float LastValue();
|
||||
|
||||
private:
|
||||
void SetAlpha(float alpha);
|
||||
|
||||
float raw_value_;
|
||||
float alpha_;
|
||||
float stored_value_;
|
||||
bool initialized_;
|
||||
};
|
||||
|
||||
} // namespace mediapipe
|
||||
|
||||
#endif // MEDIAPIPE_GRAPHS_POSE_TRACKING_CALCULATORS_LOW_PASS_FILTER_H_
|
||||
@@ -1,35 +0,0 @@
|
||||
// Copyright 2020 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "mediapipe/graphs/pose_tracking/calculators/low_pass_filter.h"
|
||||
|
||||
#include "mediapipe/framework/port/gtest.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
TEST(LowPassFilterTest, LowPassFilterBasicChecks) {
|
||||
auto filter = absl::make_unique<LowPassFilter>(1.0f);
|
||||
EXPECT_EQ(2.0f, filter->Apply(2.0f));
|
||||
EXPECT_EQ(100.0f, filter->Apply(100.0f));
|
||||
|
||||
filter = absl::make_unique<LowPassFilter>(0.0f);
|
||||
EXPECT_EQ(2.0f, filter->Apply(2.0f));
|
||||
EXPECT_EQ(2.0f, filter->Apply(100.0f));
|
||||
|
||||
filter = absl::make_unique<LowPassFilter>(0.5f);
|
||||
EXPECT_EQ(2.0f, filter->Apply(2.0f));
|
||||
EXPECT_EQ(51.0f, filter->Apply(100.0f));
|
||||
}
|
||||
|
||||
} // namespace mediapipe
|
||||
@@ -1,85 +0,0 @@
|
||||
// Copyright 2020 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "mediapipe/graphs/pose_tracking/calculators/relative_velocity_filter.h"
|
||||
|
||||
#include <cmath>
|
||||
#include <deque>
|
||||
|
||||
#include "absl/memory/memory.h"
|
||||
#include "mediapipe/framework/port/logging.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
float RelativeVelocityFilter::Apply(absl::Duration timestamp, float value_scale,
|
||||
float value) {
|
||||
const int64_t new_timestamp = absl::ToInt64Nanoseconds(timestamp);
|
||||
if (last_timestamp_ >= new_timestamp) {
|
||||
// Results are unpredictable in this case, so nothing to do but
|
||||
// return same value
|
||||
LOG(WARNING) << "New timestamp is equal or less than the last one.";
|
||||
return value;
|
||||
}
|
||||
|
||||
float alpha;
|
||||
if (last_timestamp_ == -1) {
|
||||
alpha = 1.0;
|
||||
} else {
|
||||
DCHECK(distance_mode_ == DistanceEstimationMode::kLegacyTransition ||
|
||||
distance_mode_ == DistanceEstimationMode::kForceCurrentScale);
|
||||
const float distance =
|
||||
distance_mode_ == DistanceEstimationMode::kLegacyTransition
|
||||
? value * value_scale -
|
||||
last_value_ * last_value_scale_ // Original.
|
||||
: value_scale * (value - last_value_); // Translation invariant.
|
||||
|
||||
const int64_t duration = new_timestamp - last_timestamp_;
|
||||
|
||||
float cumulative_distance = distance;
|
||||
int64_t cumulative_duration = duration;
|
||||
|
||||
// Define max cumulative duration assuming
|
||||
// 30 frames per second is a good frame rate, so assuming 30 values
|
||||
// per second or 1 / 30 of a second is a good duration per window element
|
||||
constexpr int64_t kAssumedMaxDuration = 1000000000 / 30;
|
||||
const int64_t max_cumulative_duration =
|
||||
(1 + window_.size()) * kAssumedMaxDuration;
|
||||
for (const auto& el : window_) {
|
||||
if (cumulative_duration + el.duration > max_cumulative_duration) {
|
||||
// This helps in cases when durations are large and outdated
|
||||
// window elements have bad impact on filtering results
|
||||
break;
|
||||
}
|
||||
cumulative_distance += el.distance;
|
||||
cumulative_duration += el.duration;
|
||||
}
|
||||
|
||||
constexpr double kNanoSecondsToSecond = 1e-9;
|
||||
const float velocity =
|
||||
cumulative_distance / (cumulative_duration * kNanoSecondsToSecond);
|
||||
alpha = 1.0f - 1.0f / (1.0f + velocity_scale_ * std::abs(velocity));
|
||||
window_.push_front({distance, duration});
|
||||
if (window_.size() > max_window_size_) {
|
||||
window_.pop_back();
|
||||
}
|
||||
}
|
||||
|
||||
last_value_ = value;
|
||||
last_value_scale_ = value_scale;
|
||||
last_timestamp_ = new_timestamp;
|
||||
|
||||
return low_pass_filter_.ApplyWithAlpha(value, alpha);
|
||||
}
|
||||
|
||||
} // namespace mediapipe
|
||||
@@ -1,90 +0,0 @@
|
||||
// Copyright 2020 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef MEDIAPIPE_GRAPHS_POSE_TRACKING_CALCULATORS_RELATIVE_VELOCITY_FILTER_H_
|
||||
#define MEDIAPIPE_GRAPHS_POSE_TRACKING_CALCULATORS_RELATIVE_VELOCITY_FILTER_H_
|
||||
|
||||
#include <deque>
|
||||
#include <memory>
|
||||
|
||||
#include "absl/time/time.h"
|
||||
#include "mediapipe/graphs/pose_tracking/calculators/low_pass_filter.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
// This filter keeps track (on a window of specified size) of
|
||||
// value changes over time, which as result gives us velocity of how value
|
||||
// changes over time. With higher velocity it weights new values higher.
|
||||
//
|
||||
// Use @window_size and @velocity_scale to tweak this filter for your use case.
|
||||
//
|
||||
// - higher @window_size adds to lag and to stability
|
||||
// - lower @velocity_scale adds to lag and to stability
|
||||
class RelativeVelocityFilter {
|
||||
public:
|
||||
enum class DistanceEstimationMode {
|
||||
// When the value scale changes, uses a heuristic
|
||||
// that is not translation invariant (see the implementation for details).
|
||||
kLegacyTransition,
|
||||
// The current (i.e. last) value scale is always used for scale estimation.
|
||||
// When using this mode, the filter is translation invariant, i.e.
|
||||
// Filter(Data + Offset) = Filter(Data) + Offset.
|
||||
kForceCurrentScale,
|
||||
|
||||
kDefault = kLegacyTransition
|
||||
};
|
||||
|
||||
public:
|
||||
RelativeVelocityFilter(size_t window_size, float velocity_scale,
|
||||
DistanceEstimationMode distance_mode)
|
||||
: max_window_size_{window_size},
|
||||
window_{window_size},
|
||||
velocity_scale_{velocity_scale},
|
||||
distance_mode_{distance_mode} {}
|
||||
|
||||
RelativeVelocityFilter(size_t window_size, float velocity_scale)
|
||||
: RelativeVelocityFilter{window_size, velocity_scale,
|
||||
DistanceEstimationMode::kDefault} {}
|
||||
|
||||
// Applies filter to the value.
|
||||
// @timestamp - timestamp associated with the value (for instance,
|
||||
// timestamp of the frame where you got value from)
|
||||
// @value_scale - value scale (for instance, if your value is a distance
|
||||
// detected on a frame, it can look same on different
|
||||
// devices but have quite different absolute values due
|
||||
// to different resolution, you should come up with an
|
||||
// appropriate parameter for your particular use case)
|
||||
// @value - value to filter
|
||||
float Apply(absl::Duration timestamp, float value_scale, float value);
|
||||
|
||||
private:
|
||||
struct WindowElement {
|
||||
float distance;
|
||||
int64_t duration;
|
||||
};
|
||||
|
||||
float last_value_{0.0};
|
||||
float last_value_scale_{1.0};
|
||||
int64_t last_timestamp_{-1};
|
||||
|
||||
size_t max_window_size_;
|
||||
std::deque<WindowElement> window_;
|
||||
LowPassFilter low_pass_filter_{1.0f};
|
||||
float velocity_scale_;
|
||||
DistanceEstimationMode distance_mode_;
|
||||
};
|
||||
|
||||
} // namespace mediapipe
|
||||
|
||||
#endif // MEDIAPIPE_GRAPHS_POSE_TRACKING_CALCULATORS_RELATIVE_VELOCITY_FILTER_H_
|
||||
@@ -1,292 +0,0 @@
|
||||
// Copyright 2020 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "mediapipe/graphs/pose_tracking/calculators/relative_velocity_filter.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <cmath>
|
||||
#include <vector>
|
||||
|
||||
#include "absl/time/time.h"
|
||||
#include "mediapipe/framework/port/gtest.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
using DistanceEstimationMode =
|
||||
::mediapipe::RelativeVelocityFilter::DistanceEstimationMode;
|
||||
|
||||
absl::Duration DurationFromNanos(int64_t nanos) {
|
||||
return absl::FromChrono(std::chrono::nanoseconds{nanos});
|
||||
}
|
||||
|
||||
absl::Duration DurationFromMillis(int64_t millis) {
|
||||
return absl::FromChrono(std::chrono::milliseconds{millis});
|
||||
}
|
||||
|
||||
TEST(RelativeVelocityFilterTest, ApplyIncorrectTimestamp) {
|
||||
auto filter = absl::make_unique<RelativeVelocityFilter>(1, 1.0);
|
||||
|
||||
absl::Duration timestamp1 = DurationFromNanos(1);
|
||||
|
||||
EXPECT_FLOAT_EQ(95.5f, filter->Apply(timestamp1, 0.5f, 95.5f));
|
||||
EXPECT_FLOAT_EQ(200.5f, filter->Apply(timestamp1, 0.5f, 200.5f));
|
||||
EXPECT_FLOAT_EQ(1000.5f, filter->Apply(timestamp1, 0.5f, 1000.5f));
|
||||
|
||||
EXPECT_FLOAT_EQ(2000.0f, filter->Apply(DurationFromNanos(1), 0.5f, 2000.0f));
|
||||
}
|
||||
|
||||
void TestSameValueScaleDifferentVelocityScales(
|
||||
DistanceEstimationMode distance_mode) {
|
||||
// Changing the distance estimation mode has no effect with constant scales.
|
||||
|
||||
// More sensitive filter.
|
||||
auto filter1 = absl::make_unique<RelativeVelocityFilter>(
|
||||
/*window_size=*/5, /*velocity_scale=*/45.0f,
|
||||
/*distance_mode=*/distance_mode);
|
||||
// Less sensitive filter.
|
||||
auto filter2 = absl::make_unique<RelativeVelocityFilter>(
|
||||
/*window_size=*/5, /*velocity_scale=*/0.1f,
|
||||
/*distance_mode=*/distance_mode);
|
||||
|
||||
float result1;
|
||||
float result2;
|
||||
float value;
|
||||
float value_scale = 1.0f;
|
||||
|
||||
value = 1.0f;
|
||||
result1 = filter1->Apply(DurationFromMillis(1), value_scale, value);
|
||||
result2 = filter2->Apply(DurationFromMillis(1), value_scale, value);
|
||||
EXPECT_EQ(result1, result2);
|
||||
|
||||
value = 10.0f;
|
||||
result1 = filter1->Apply(DurationFromMillis(2), value_scale, value);
|
||||
result2 = filter2->Apply(DurationFromMillis(2), value_scale, value);
|
||||
EXPECT_GT(result1, result2);
|
||||
|
||||
value = 2.0f;
|
||||
result1 = filter1->Apply(DurationFromMillis(3), value_scale, value);
|
||||
result2 = filter2->Apply(DurationFromMillis(3), value_scale, value);
|
||||
EXPECT_LT(result1, result2);
|
||||
|
||||
value = 20.0f;
|
||||
result1 = filter1->Apply(DurationFromMillis(4), value_scale, value);
|
||||
result2 = filter2->Apply(DurationFromMillis(4), value_scale, value);
|
||||
EXPECT_GT(result1, result2);
|
||||
|
||||
value = 10.0f;
|
||||
result1 = filter1->Apply(DurationFromMillis(5), value_scale, value);
|
||||
result2 = filter2->Apply(DurationFromMillis(5), value_scale, value);
|
||||
EXPECT_LT(result1, result2);
|
||||
|
||||
value = 50.0f;
|
||||
result1 = filter1->Apply(DurationFromMillis(6), value_scale, value);
|
||||
result2 = filter2->Apply(DurationFromMillis(6), value_scale, value);
|
||||
EXPECT_GT(result1, result2);
|
||||
|
||||
value = 30.0f;
|
||||
result1 = filter1->Apply(DurationFromMillis(7), value_scale, value);
|
||||
result2 = filter2->Apply(DurationFromMillis(7), value_scale, value);
|
||||
EXPECT_LT(result1, result2);
|
||||
}
|
||||
|
||||
TEST(RelativeVelocityFilterTest, SameValueScaleDifferentVelocityScalesLegacy) {
|
||||
TestSameValueScaleDifferentVelocityScales(
|
||||
DistanceEstimationMode::kLegacyTransition);
|
||||
}
|
||||
|
||||
TEST(RelativeVelocityFilterTest,
|
||||
SameValueScaleDifferentVelocityScalesForceCurrentScale) {
|
||||
TestSameValueScaleDifferentVelocityScales(
|
||||
DistanceEstimationMode::kForceCurrentScale);
|
||||
}
|
||||
|
||||
void TestDifferentConstantValueScalesSameVelocityScale(
|
||||
DistanceEstimationMode distance_mode) {
|
||||
const float same_velocity_scale = 1.0f;
|
||||
auto filter1 = absl::make_unique<RelativeVelocityFilter>(
|
||||
/*window_size=*/3, /*velocity_scale=*/same_velocity_scale,
|
||||
/*distance_mode=*/distance_mode);
|
||||
auto filter2 = absl::make_unique<RelativeVelocityFilter>(
|
||||
/*window_size=*/3, /*velocity_scale=*/same_velocity_scale,
|
||||
/*distance_mode=*/distance_mode);
|
||||
|
||||
float result1;
|
||||
float result2;
|
||||
float value;
|
||||
// smaller value scale will decrease cumulative speed and alpha
|
||||
// so with smaller scale and same other params filter will believe
|
||||
// new values a little bit less
|
||||
float value_scale1 = 0.5f;
|
||||
float value_scale2 = 1.0f;
|
||||
|
||||
value = 1.0f;
|
||||
result1 = filter1->Apply(DurationFromMillis(1), value_scale1, value);
|
||||
result2 = filter2->Apply(DurationFromMillis(1), value_scale2, value);
|
||||
EXPECT_EQ(result1, result2);
|
||||
|
||||
value = 10.0f;
|
||||
result1 = filter1->Apply(DurationFromMillis(2), value_scale1, value);
|
||||
result2 = filter2->Apply(DurationFromMillis(2), value_scale2, value);
|
||||
EXPECT_LT(result1, result2);
|
||||
|
||||
value = 2.0f;
|
||||
result1 = filter1->Apply(DurationFromMillis(3), value_scale1, value);
|
||||
result2 = filter2->Apply(DurationFromMillis(3), value_scale2, value);
|
||||
EXPECT_GT(result1, result2);
|
||||
|
||||
value = 20.0f;
|
||||
result1 = filter1->Apply(DurationFromMillis(4), value_scale1, value);
|
||||
result2 = filter2->Apply(DurationFromMillis(4), value_scale2, value);
|
||||
EXPECT_LT(result1, result2);
|
||||
}
|
||||
|
||||
TEST(RelativeVelocityFilterTest,
|
||||
DifferentConstantValueScalesSameVelocityScale) {
|
||||
TestDifferentConstantValueScalesSameVelocityScale(
|
||||
DistanceEstimationMode::kLegacyTransition);
|
||||
}
|
||||
|
||||
TEST(RelativeVelocityFilterTest, ApplyCheckValueScales) {
|
||||
TestDifferentConstantValueScalesSameVelocityScale(
|
||||
DistanceEstimationMode::kForceCurrentScale);
|
||||
}
|
||||
|
||||
void TestTranslationInvariance(DistanceEstimationMode distance_mode) {
|
||||
struct ValueAtScale {
|
||||
float value;
|
||||
float scale;
|
||||
};
|
||||
|
||||
// Note that the scales change over time.
|
||||
std::vector<ValueAtScale> original_data_points{
|
||||
// clang-format off
|
||||
{.value = 1.0f, .scale = 0.5f},
|
||||
{.value = 10.0f, .scale = 5.0f},
|
||||
{.value = 20.0f, .scale = 10.0f},
|
||||
{.value = 30.0f, .scale = 15.0f},
|
||||
{.value = 40.0f, .scale = 0.5f},
|
||||
{.value = 50.0f, .scale = 0.5f},
|
||||
{.value = 60.0f, .scale = 5.0f},
|
||||
{.value = 70.0f, .scale = 10.0f},
|
||||
{.value = 80.0f, .scale = 15.0f},
|
||||
{.value = 90.0f, .scale = 5.0f},
|
||||
{.value = 70.0f, .scale = 10.0f},
|
||||
{.value = 50.0f, .scale = 15.0f},
|
||||
{.value = 80.0f, .scale = 15.0f},
|
||||
// clang-format on
|
||||
};
|
||||
|
||||
// The amount by which the input values are uniformly translated.
|
||||
const float kValueOffset = 100.0f;
|
||||
|
||||
// The uniform time delta.
|
||||
const absl::Duration time_delta = DurationFromMillis(1);
|
||||
|
||||
// The filter parameters are the same between the two filters.
|
||||
const size_t kWindowSize = 5;
|
||||
const float kVelocityScale = 0.1f;
|
||||
|
||||
// Perform the translation.
|
||||
std::vector<ValueAtScale> translated_data_points = original_data_points;
|
||||
for (auto& point : translated_data_points) {
|
||||
point.value += kValueOffset;
|
||||
}
|
||||
|
||||
auto original_points_filter = absl::make_unique<RelativeVelocityFilter>(
|
||||
/*window_size=*/kWindowSize, /*velocity_scale=*/kVelocityScale,
|
||||
/*distance_mode=*/distance_mode);
|
||||
auto translated_points_filter = absl::make_unique<RelativeVelocityFilter>(
|
||||
/*window_size=*/kWindowSize, /*velocity_scale=*/kVelocityScale,
|
||||
/*distance_mode=*/distance_mode);
|
||||
|
||||
// The minimal difference which is considered a divergence.
|
||||
const float kDivergenceGap = 0.001f;
|
||||
// The amount of the times this gap is achieved with `kLegacyTransition`.
|
||||
// Note that on the first iteration the filters should output the unfiltered
|
||||
// input values, so no divergence should occur.
|
||||
// This amount obviously depends on the values in `original_data_points`,
|
||||
// so should be changed accordingly when they are updated.
|
||||
const size_t kDivergenceTimes = 5;
|
||||
|
||||
// The minimal difference which is considered a large divergence.
|
||||
const float kLargeDivergenceGap = 10.0f;
|
||||
// The amount of times it is achieved.
|
||||
// This amount obviously depends on the values in `original_data_points`,
|
||||
// so should be changed accordingly when they are updated.
|
||||
const size_t kLargeDivergenceTimes = 1;
|
||||
|
||||
// In contrast, the new mode delivers this error bound across all the samples.
|
||||
const float kForceCurrentScaleAbsoluteError = 1.53e-05f;
|
||||
|
||||
size_t times_diverged = 0;
|
||||
size_t times_largely_diverged = 0;
|
||||
absl::Duration timestamp;
|
||||
for (size_t iteration = 0; iteration < original_data_points.size();
|
||||
++iteration, timestamp += time_delta) {
|
||||
const ValueAtScale& original_data_point = original_data_points[iteration];
|
||||
const float filtered_original_value =
|
||||
original_points_filter->Apply(/*timestamp=*/timestamp,
|
||||
/*value_scale=*/original_data_point.scale,
|
||||
/*value=*/original_data_point.value);
|
||||
|
||||
const ValueAtScale& translated_data_point =
|
||||
translated_data_points[iteration];
|
||||
const float actual_filtered_translated_value =
|
||||
translated_points_filter->Apply(
|
||||
/*timestamp=*/timestamp,
|
||||
/*value_scale=*/translated_data_point.scale,
|
||||
/*value=*/translated_data_point.value);
|
||||
|
||||
const float expected_filtered_translated_value =
|
||||
filtered_original_value + kValueOffset;
|
||||
|
||||
const float difference = std::fabs(actual_filtered_translated_value -
|
||||
expected_filtered_translated_value);
|
||||
if (iteration == 0) {
|
||||
// On the first iteration, the unfiltered values are returned.
|
||||
EXPECT_EQ(filtered_original_value, original_data_point.value);
|
||||
EXPECT_EQ(actual_filtered_translated_value, translated_data_point.value);
|
||||
EXPECT_EQ(difference, 0.0f);
|
||||
} else if (distance_mode == DistanceEstimationMode::kLegacyTransition) {
|
||||
if (difference >= kDivergenceGap) {
|
||||
++times_diverged;
|
||||
}
|
||||
if (difference >= kLargeDivergenceGap) {
|
||||
++times_largely_diverged;
|
||||
}
|
||||
} else {
|
||||
CHECK(distance_mode == DistanceEstimationMode::kForceCurrentScale);
|
||||
EXPECT_NEAR(difference, 0.0f, kForceCurrentScaleAbsoluteError);
|
||||
}
|
||||
}
|
||||
|
||||
if (distance_mode == DistanceEstimationMode::kLegacyTransition) {
|
||||
EXPECT_GE(times_diverged, kDivergenceTimes);
|
||||
EXPECT_GE(times_largely_diverged, kLargeDivergenceTimes);
|
||||
}
|
||||
}
|
||||
|
||||
// This test showcases an undesired property of the current filter design
|
||||
// that manifests itself when value scales change in time. It turns out that
|
||||
// the velocity estimation starts depending on the distance from the origin.
|
||||
TEST(RelativeVelocityFilterTest,
|
||||
TestLegacyFilterModeIsNotTranslationInvariant) {
|
||||
TestTranslationInvariance(DistanceEstimationMode::kLegacyTransition);
|
||||
}
|
||||
|
||||
TEST(RelativeVelocityFilterTest, TestOtherFilterModeIsTranslationInvariant) {
|
||||
TestTranslationInvariance(DistanceEstimationMode::kForceCurrentScale);
|
||||
}
|
||||
|
||||
} // namespace mediapipe
|
||||
Reference in New Issue
Block a user