talksik/mediapipe
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Project import generated by Copybara.

Browse Source 
GitOrigin-RevId: f72a0f86c2c2acdb1920973c718a9e26ed3ec4b6
This commit is contained in:
MediaPipe Team
2020-06-08 12:08:33 -04:00
committed by chuoling
parent 5d028d923b
commit cd2b69d58c
315 changed files with 8162 additions and 7502 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mediapipe/calculators/core/split_normalized_landmark_list_calculator.cc
+3 -1
View File
@@ -112,7 +112,9 @@ class SplitNormalizedLandmarkListCalculator : public CalculatorBase {
::mediapipe::Status Process(CalculatorContext* cc) override {
const NormalizedLandmarkList& input =
cc->Inputs().Index(0).Get<NormalizedLandmarkList>();
RET_CHECK_GE(input.landmark_size(), max_range_end_);
RET_CHECK_GE(input.landmark_size(), max_range_end_)
<< "Max range end " << max_range_end_ << " exceeds landmarks size "
<< input.landmark_size();
if (combine_outputs_) {
NormalizedLandmarkList output;
mediapipe/calculators/image/BUILD
+1
View File
@@ -258,6 +258,7 @@ cc_library(
":bilateral_filter_calculator_cc_proto",
"//mediapipe/framework:calculator_options_cc_proto",
"//mediapipe/framework/formats:image_format_cc_proto",
"@com_google_absl//absl/strings",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/formats:image_frame_opencv",
mediapipe/calculators/image/bilateral_filter_calculator.cc
+87 -124
View File
@@ -15,6 +15,7 @@
#include <memory>
#include <string>
#include "absl/strings/str_replace.h"
#include "mediapipe/calculators/image/bilateral_filter_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_options.pb.h"
@@ -104,8 +105,9 @@ class BilateralFilterCalculator : public CalculatorBase {
#if !defined(MEDIAPIPE_DISABLE_GPU)
mediapipe::GlCalculatorHelper gpu_helper_;
GLuint program_ = 0;
GLuint program_joint_ = 0;
#endif // !MEDIAPIPE_DISABLE_GPU
GLuint vao_;
GLuint vbo_[2]; // vertex storage
#endif // !MEDIAPIPE_DISABLE_GPU
};
REGISTER_CALCULATOR(BilateralFilterCalculator);
@@ -219,9 +221,12 @@ REGISTER_CALCULATOR(BilateralFilterCalculator);
#if !defined(MEDIAPIPE_DISABLE_GPU)
gpu_helper_.RunInGlContext([this] {
if (program_) glDeleteProgram(program_);
if (vao_) glDeleteVertexArrays(1, &vao_);
if (vbo_[0]) glDeleteBuffers(2, vbo_);
program_ = 0;
if (program_joint_) glDeleteProgram(program_joint_);
program_joint_ = 0;
vao_ = 0;
vbo_[0] = 0;
vbo_[1] = 0;
});
#endif // !MEDIAPIPE_DISABLE_GPU
@@ -276,17 +281,18 @@ REGISTER_CALCULATOR(BilateralFilterCalculator);
auto input_texture = gpu_helper_.CreateSourceTexture(input_frame);
mediapipe::GlTexture output_texture;
const bool has_guide_image = cc->Inputs().HasTag(kInputGuideTagGpu) &&
!cc->Inputs().Tag(kInputGuideTagGpu).IsEmpty();
const bool has_guide_image = cc->Inputs().HasTag(kInputGuideTagGpu);
// Setup textures and Update image in GPU shader.
if (has_guide_image) {
if (cc->Inputs().Tag(kInputGuideTagGpu).IsEmpty())
return mediapipe::OkStatus();
// joint bilateral filter
glUseProgram(program_joint_);
glUseProgram(program_);
const auto& guide_image =
cc->Inputs().Tag(kInputGuideTagGpu).Get<mediapipe::GpuBuffer>();
auto guide_texture = gpu_helper_.CreateSourceTexture(guide_image);
glUniform2f(glGetUniformLocation(program_joint_, "texel_size_guide"),
glUniform2f(glGetUniformLocation(program_, "texel_size_guide"),
1.0 / guide_image.width(), 1.0 / guide_image.height());
output_texture = gpu_helper_.CreateDestinationTexture(
guide_image.width(), guide_image.height(),
@@ -297,7 +303,6 @@ REGISTER_CALCULATOR(BilateralFilterCalculator);
glActiveTexture(GL_TEXTURE2);
glBindTexture(GL_TEXTURE_2D, guide_texture.name());
GlRender(cc);
glActiveTexture(GL_TEXTURE2);
glBindTexture(GL_TEXTURE_2D, 0);
glActiveTexture(GL_TEXTURE1);
glBindTexture(GL_TEXTURE_2D, 0);
@@ -314,7 +319,6 @@ REGISTER_CALCULATOR(BilateralFilterCalculator);
glActiveTexture(GL_TEXTURE1);
glBindTexture(GL_TEXTURE_2D, input_texture.name());
GlRender(cc);
glActiveTexture(GL_TEXTURE1);
glBindTexture(GL_TEXTURE_2D, 0);
}
glFlush();
@@ -335,51 +339,14 @@ REGISTER_CALCULATOR(BilateralFilterCalculator);
void BilateralFilterCalculator::GlRender(CalculatorContext* cc) {
#if !defined(MEDIAPIPE_DISABLE_GPU)
static const GLfloat square_vertices[] = {
-1.0f, -1.0f, // bottom left
1.0f, -1.0f, // bottom right
-1.0f, 1.0f, // top left
1.0f, 1.0f, // top right
};
static const GLfloat texture_vertices[] = {
0.0f, 0.0f, // bottom left
1.0f, 0.0f, // bottom right
0.0f, 1.0f, // top left
1.0f, 1.0f, // top right
};
// vertex storage
GLuint vbo[2];
glGenBuffers(2, vbo);
GLuint vao;
glGenVertexArrays(1, &vao);
glBindVertexArray(vao);
// vbo 0
glBindBuffer(GL_ARRAY_BUFFER, vbo[0]);
glBufferData(GL_ARRAY_BUFFER, 4 * 2 * sizeof(GLfloat), square_vertices,
GL_STATIC_DRAW);
glEnableVertexAttribArray(ATTRIB_VERTEX);
glVertexAttribPointer(ATTRIB_VERTEX, 2, GL_FLOAT, 0, 0, nullptr);
// vbo 1
glBindBuffer(GL_ARRAY_BUFFER, vbo[1]);
glBufferData(GL_ARRAY_BUFFER, 4 * 2 * sizeof(GLfloat), texture_vertices,
GL_STATIC_DRAW);
glEnableVertexAttribArray(ATTRIB_TEXTURE_POSITION);
glVertexAttribPointer(ATTRIB_TEXTURE_POSITION, 2, GL_FLOAT, 0, 0, nullptr);
// bring back vao and vbo
glBindVertexArray(vao_);
// draw
glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
// cleanup
glDisableVertexAttribArray(ATTRIB_VERTEX);
glDisableVertexAttribArray(ATTRIB_TEXTURE_POSITION);
glBindBuffer(GL_ARRAY_BUFFER, 0);
glBindVertexArray(0);
glDeleteVertexArrays(1, &vao);
glDeleteBuffers(2, vbo);
#endif // !MEDIAPIPE_DISABLE_GPU
}
@@ -394,36 +361,11 @@ void BilateralFilterCalculator::GlRender(CalculatorContext* cc) {
"texture_coordinate",
};
// We bake our sigma values directly into the shader, so the GLSL compiler can
// optimize appropriately.
std::string sigma_options_string =
"const float sigma_space = " + std::to_string(sigma_space_) +
"; const float sigma_color = " + std::to_string(sigma_color_) + ";\n";
// Shader to do bilateral filtering on input image based on sigma space/color.
// Large kernel sizes are subsampled based on sqrt(sigma_space) window size,
// denoted as 'sparsity' below.
const std::string frag_src = GLES_VERSION_COMPAT
R"(
#if __VERSION__ < 130
#define in varying
#endif // __VERSION__ < 130
#ifdef GL_ES
#define fragColor gl_FragColor
precision highp float;
#else
#define lowp
#define mediump
#define highp
#define texture2D texture
out vec4 fragColor;
#endif // defined(GL_ES)
in vec2 sample_coordinate;
uniform sampler2D input_frame;
)" + sigma_options_string + R"(
uniform vec2 texel_size;
// Common functions and settings for both shaders.
const std::string common_string =
absl::StrReplaceAll(R"(
const float sigma_space = $space;
const float sigma_color = $color;
const float kSparsityFactor = 0.66; // Higher is more sparse.
const float sparsity = max(1.0, sqrt(sigma_space) * kSparsityFactor);
@@ -435,6 +377,23 @@ void BilateralFilterCalculator::GlRender(CalculatorContext* cc) {
float coeff = -0.5 / (sigma * sigma * 4.0 + 1.0e-6);
return exp((x * x) * coeff);
}
)",
{{"$space", std::to_string(sigma_space_)},
{"$color", std::to_string(sigma_color_)}});
// Shader to do bilateral filtering on input image based on sigma space/color.
// Large kernel sizes are subsampled based on sqrt(sigma_space) window size,
// denoted as 'sparsity' below.
const std::string frag_src =
std::string(mediapipe::kMediaPipeFragmentShaderPreamble) + R"(
DEFAULT_PRECISION(highp, float)
in vec2 sample_coordinate;
uniform sampler2D input_frame;
uniform vec2 texel_size;
)" +
common_string + R"(
void main() {
vec2 center_uv = sample_coordinate;
@@ -462,55 +421,25 @@ void BilateralFilterCalculator::GlRender(CalculatorContext* cc) {
}
new_val /= vec3(total_weight);
fragColor = vec4(new_val, 1.0);
gl_FragColor = vec4(new_val, 1.0);
}
)";
// Create shader program and set parameters.
mediapipe::GlhCreateProgram(mediapipe::kBasicVertexShader, frag_src.c_str(),
NUM_ATTRIBUTES, (const GLchar**)&attr_name[0],
attr_location, &program_);
RET_CHECK(program_) << "Problem initializing the program.";
glUseProgram(program_);
glUniform1i(glGetUniformLocation(program_, "input_frame"), 1);
// Shader to do joint bilateral filtering on input image based on
// sigma space/color, and a Guide image.
// Large kernel sizes are subsampled based on sqrt(sigma_space) window size,
// denoted as 'sparsity' below.
const std::string joint_frag_src = GLES_VERSION_COMPAT
R"(
#if __VERSION__ < 130
#define in varying
#endif // __VERSION__ < 130
#ifdef GL_ES
#define fragColor gl_FragColor
precision highp float;
#else
#define lowp
#define mediump
#define highp
#define texture2D texture
out vec4 fragColor;
#endif // defined(GL_ES)
const std::string joint_frag_src =
std::string(mediapipe::kMediaPipeFragmentShaderPreamble) + R"(
DEFAULT_PRECISION(highp, float)
in vec2 sample_coordinate;
uniform sampler2D input_frame;
uniform sampler2D guide_frame;
)" + sigma_options_string + R"(
uniform vec2 texel_size_guide; // size of guide and resulting filtered image
const float kSparsityFactor = 0.66; // Higher is more sparse.
const float sparsity = max(1.0, sqrt(sigma_space) * kSparsityFactor);
const float step = sparsity;
const float radius = sigma_space;
const float offset = (step > 1.0) ? (step * 0.5) : (0.0);
float gaussian(float x, float sigma) {
float coeff = -0.5 / (sigma * sigma * 4.0 + 1.0e-6);
return exp((x * x) * coeff);
}
)" +
common_string + R"(
void main() {
vec2 center_uv = sample_coordinate;
@@ -539,18 +468,52 @@ void BilateralFilterCalculator::GlRender(CalculatorContext* cc) {
}
new_val /= vec3(total_weight);
fragColor = vec4(new_val, 1.0);
gl_FragColor = vec4(new_val, 1.0);
}
)";
// Create shader program and set parameters.
mediapipe::GlhCreateProgram(
mediapipe::kBasicVertexShader, joint_frag_src.c_str(), NUM_ATTRIBUTES,
(const GLchar**)&attr_name[0], attr_location, &program_joint_);
RET_CHECK(program_joint_) << "Problem initializing the program.";
glUseProgram(program_joint_);
glUniform1i(glGetUniformLocation(program_joint_, "input_frame"), 1);
glUniform1i(glGetUniformLocation(program_joint_, "guide_frame"), 2);
// Only initialize the one shader to be used.
const bool has_guide_image = cc->Inputs().HasTag(kInputGuideTagGpu);
if (has_guide_image) {
// Create joint shader program and set parameters.
mediapipe::GlhCreateProgram(
mediapipe::kBasicVertexShader, joint_frag_src.c_str(), NUM_ATTRIBUTES,
(const GLchar**)&attr_name[0], attr_location, &program_);
RET_CHECK(program_) << "Problem initializing the program.";
glUseProgram(program_);
glUniform1i(glGetUniformLocation(program_, "input_frame"), 1);
glUniform1i(glGetUniformLocation(program_, "guide_frame"), 2);
} else {
// Create default shader program and set parameters.
mediapipe::GlhCreateProgram(mediapipe::kBasicVertexShader, frag_src.c_str(),
NUM_ATTRIBUTES, (const GLchar**)&attr_name[0],
attr_location, &program_);
RET_CHECK(program_) << "Problem initializing the program.";
glUseProgram(program_);
glUniform1i(glGetUniformLocation(program_, "input_frame"), 1);
}
// Generate vbos and vao.
glGenVertexArrays(1, &vao_);
glGenBuffers(2, vbo_);
// Fill in static vbo (vbo 0), to be reused in GlRender().
glBindVertexArray(vao_);
glBindBuffer(GL_ARRAY_BUFFER, vbo_[0]);
glBufferData(GL_ARRAY_BUFFER, 4 * 2 * sizeof(GLfloat),
mediapipe::kBasicSquareVertices, GL_STATIC_DRAW);
glEnableVertexAttribArray(ATTRIB_VERTEX);
glVertexAttribPointer(ATTRIB_VERTEX, 2, GL_FLOAT, 0, 0, nullptr);
glBindBuffer(GL_ARRAY_BUFFER, 0);
// Fill in static vbo (vbo 1), to be reused in GlRender().
glBindBuffer(GL_ARRAY_BUFFER, vbo_[1]);
glBufferData(GL_ARRAY_BUFFER, 4 * 2 * sizeof(GLfloat),
mediapipe::kBasicTextureVertices, GL_STATIC_DRAW);
glEnableVertexAttribArray(ATTRIB_TEXTURE_POSITION);
glVertexAttribPointer(ATTRIB_TEXTURE_POSITION, 2, GL_FLOAT, 0, 0, nullptr);
glBindBuffer(GL_ARRAY_BUFFER, 0);
glBindVertexArray(0);
#endif // !MEDIAPIPE_DISABLE_GPU
mediapipe/calculators/image/image_transformation_calculator.cc
+1
View File
@@ -570,6 +570,7 @@ void ImageTransformationCalculator::ComputeOutputDimensions(
void ImageTransformationCalculator::ComputeOutputLetterboxPadding(
int input_width, int input_height, int output_width, int output_height,
std::array<float, 4>* padding) {
padding->fill(0.f);
if (scale_mode_ == mediapipe::ScaleMode_Mode_FIT) {
if (rotation_ == mediapipe::RotationMode_Mode_ROTATION_90 ||
rotation_ == mediapipe::RotationMode_Mode_ROTATION_270) {
mediapipe/calculators/tensorflow/vector_int_to_tensor_calculator_test.cc
+3 -2
View File
@@ -153,7 +153,7 @@ TEST_F(VectorIntToTensorCalculatorTest, TestInt64) {
const int64 time = 1234;
runner_->MutableInputs()
->Tag("SINGLE_INT")
.packets.push_back(MakePacket<int>(2 ^ 31).At(Timestamp(time)));
.packets.push_back(MakePacket<int>(1LL << 31).At(Timestamp(time)));
EXPECT_TRUE(runner_->Run().ok());
@@ -166,7 +166,8 @@ TEST_F(VectorIntToTensorCalculatorTest, TestInt64) {
EXPECT_EQ(1, output_tensor.dims());
EXPECT_EQ(tf::DT_INT64, output_tensor.dtype());
const auto vec = output_tensor.vec<tf::int64>();
EXPECT_EQ(2 ^ 31, vec(0));
// 1LL << 31 overflows the positive int and becomes negative.
EXPECT_EQ(static_cast<int>(1LL << 31), vec(0));
}
TEST_F(VectorIntToTensorCalculatorTest, TestUint8) {
mediapipe/calculators/tflite/tflite_inference_calculator.cc
+7
View File
@@ -358,6 +358,13 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
cc->Options<mediapipe::TfLiteInferenceCalculatorOptions>();
use_advanced_gpu_api_ = false;
if (use_advanced_gpu_api_ && !(gpu_input_ && gpu_output_)) {
LOG(WARNING)
<< "Cannot use advanced GPU APIs, both inputs and outputs must "
"be GPU buffers. Falling back to the default TFLite API.";
use_advanced_gpu_api_ = false;
}
MP_RETURN_IF_ERROR(LoadModel(cc));
if (gpu_inference_) {
mediapipe/calculators/tflite/tflite_tensors_to_landmarks_calculator.cc
+7 -3
View File
@@ -21,8 +21,9 @@
namespace mediapipe {
// A calculator for converting TFLite tensors from regression models into
// landmarks. Note that if the landmarks in the tensor has more than 3
// dimensions, only the first 3 dimensions will be converted to x,y,z.
// landmarks. Note that if the landmarks in the tensor has more than 4
// dimensions, only the first 4 dimensions will be converted to
// [x,y,z, visibility].
//
// Input:
// TENSORS - Vector of TfLiteTensor of type kTfLiteFloat32. Only the first
@@ -205,12 +206,14 @@ REGISTER_CALCULATOR(TfLiteTensorsToLandmarksCalculator);
if (num_dimensions > 2) {
landmark->set_z(raw_landmarks[offset + 2]);
}
if (num_dimensions > 3) {
landmark->set_visibility(raw_landmarks[offset + 3]);
}
}
// Output normalized landmarks if required.
if (cc->Outputs().HasTag("NORM_LANDMARKS")) {
NormalizedLandmarkList output_norm_landmarks;
// for (const auto& landmark : output_landmarks) {
for (int i = 0; i < output_landmarks.landmark_size(); ++i) {
const Landmark& landmark = output_landmarks.landmark(i);
NormalizedLandmark* norm_landmark = output_norm_landmarks.add_landmark();
@@ -219,6 +222,7 @@ REGISTER_CALCULATOR(TfLiteTensorsToLandmarksCalculator);
norm_landmark->set_y(static_cast<float>(landmark.y()) /
options_.input_image_height());
norm_landmark->set_z(landmark.z() / options_.normalize_z());
norm_landmark->set_visibility(landmark.visibility());
}
cc->Outputs()
.Tag("NORM_LANDMARKS")
mediapipe/calculators/util/BUILD
+15
View File
@@ -320,6 +320,7 @@ cc_library(
"//mediapipe/framework:calculator_options_cc_proto",
"//mediapipe/framework/formats:image_format_cc_proto",
"//mediapipe/util:color_cc_proto",
"@com_google_absl//absl/strings",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/formats:video_stream_header",
@@ -541,6 +542,19 @@ cc_library(
alwayslink = 1,
)
cc_library(
name = "rect_projection_calculator",
srcs = ["rect_projection_calculator.cc"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/formats:rect_cc_proto",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
],
alwayslink = 1,
)
cc_test(
name = "detections_to_rects_calculator_test",
size = "small",
@@ -951,6 +965,7 @@ cc_library(
visibility = ["//visibility:public"],
deps = [
":top_k_scores_calculator_cc_proto",
"@com_google_absl//absl/container:node_hash_map",
"//mediapipe/framework/formats:classification_cc_proto",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
mediapipe/calculators/util/annotation_overlay_calculator.cc
+26 -13
View File
@@ -14,6 +14,7 @@
#include <memory>
#include "absl/strings/str_cat.h"
#include "mediapipe/calculators/util/annotation_overlay_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_options.pb.h"
@@ -573,31 +574,33 @@ REGISTER_CALCULATOR(AnnotationOverlayCalculator);
};
// Shader to overlay a texture onto another when overlay is non-zero.
const GLchar* frag_src = GLES_VERSION_COMPAT
R"(
#if __VERSION__ < 130
#define in varying
#endif // __VERSION__ < 130
constexpr char kFragSrcBody[] = R"(
DEFAULT_PRECISION(mediump, float)
#ifdef GL_ES
#define fragColor gl_FragColor
precision highp float;
#else
#define lowp
#define mediump
#define highp
#define texture2D texture
out vec4 fragColor;
#endif // defined(GL_ES)
#endif // GL_ES
in vec2 sample_coordinate;
uniform sampler2D input_frame;
// "overlay" texture has top-left origin (OpenCV mat with annotations has
// been uploaded to GPU without vertical flip)
uniform sampler2D overlay;
uniform vec3 transparent_color;
void main() {
vec3 image_pix = texture2D(input_frame, sample_coordinate).rgb;
#ifdef INPUT_FRAME_HAS_TOP_LEFT_ORIGIN
// "input_frame" has top-left origin same as "overlay", hence overlaying
// as is.
vec3 overlay_pix = texture2D(overlay, sample_coordinate).rgb;
#else
// "input_frame" has bottom-left origin, hence flipping "overlay" texture
// coordinates.
vec3 overlay_pix = texture2D(overlay, vec2(sample_coordinate.x, 1.0 - sample_coordinate.y)).rgb;
#endif // INPUT_FRAME_HAS_TOP_LEFT_ORIGIN
vec3 out_pix = image_pix;
float dist = distance(overlay_pix.rgb, transparent_color);
if (dist > 0.001) out_pix = overlay_pix;
@@ -606,8 +609,18 @@ REGISTER_CALCULATOR(AnnotationOverlayCalculator);
}
)";
std::string defines;
if (options_.gpu_uses_top_left_origin()) {
defines = R"(
#define INPUT_FRAME_HAS_TOP_LEFT_ORIGIN;
)";
}
const std::string frag_src = absl::StrCat(
mediapipe::kMediaPipeFragmentShaderPreamble, defines, kFragSrcBody);
// Create shader program and set parameters
mediapipe::GlhCreateProgram(mediapipe::kBasicVertexShader, frag_src,
mediapipe::GlhCreateProgram(mediapipe::kBasicVertexShader, frag_src.c_str(),
NUM_ATTRIBUTES, (const GLchar**)&attr_name[0],
attr_location, &program_);
RET_CHECK(program_) << "Problem initializing the program.";
mediapipe/calculators/util/annotation_overlay_calculator.proto
+5
View File
@@ -40,4 +40,9 @@ message AnnotationOverlayCalculatorOptions {
// top-left corner. Therefore, for images with the origin at the bottom-left
// corner this should be set to true.
optional bool flip_text_vertically = 5 [default = false];
// Whether input stream IMAGE_GPU (OpenGL texture) has bottom-left or top-left
// origin. (Historically, OpenGL uses bottom left origin, but most MediaPipe
// examples expect textures to have top-left origin.)
optional bool gpu_uses_top_left_origin = 6 [default = true];
}
mediapipe/calculators/util/landmark_letterbox_removal_calculator.cc
+2
View File
@@ -127,6 +127,8 @@ class LandmarkLetterboxRemovalCalculator : public CalculatorBase {
new_landmark->set_y(new_y);
// Keep z-coord as is.
new_landmark->set_z(landmark.z());
// Keep visibility as is.
new_landmark->set_visibility(landmark.visibility());
}
cc->Outputs().Get(output_id).AddPacket(
mediapipe/calculators/util/landmark_projection_calculator.cc
+2
View File
@@ -128,6 +128,8 @@ class LandmarkProjectionCalculator : public CalculatorBase {
new_landmark->set_y(new_y);
// Keep z-coord as is.
new_landmark->set_z(landmark.z());
// Keep visibility as is.
new_landmark->set_visibility(landmark.visibility());
}
cc->Outputs().Get(output_id).AddPacket(
mediapipe/calculators/util/landmarks_to_render_data_calculator.cc
+44 -6
View File
@@ -97,11 +97,17 @@ void AddConnectionToRenderData(const LandmarkType& start,
template <class LandmarkListType, class LandmarkType>
void AddConnectionsWithDepth(const LandmarkListType& landmarks,
const std::vector<int>& landmark_connections,
float thickness, bool normalized, float min_z,
float max_z, RenderData* render_data) {
bool utilize_visibility,
float visibility_threshold, float thickness,
bool normalized, float min_z, float max_z,
RenderData* render_data) {
for (int i = 0; i < landmark_connections.size(); i += 2) {
const auto& ld0 = landmarks.landmark(landmark_connections[i]);
const auto& ld1 = landmarks.landmark(landmark_connections[i + 1]);
if (visibility_threshold && (ld0.visibility() < visibility_threshold ||
ld1.visibility() < visibility_threshold)) {
continue;
}
const int gray_val1 =
255 - static_cast<int>(Remap(ld0.z(), min_z, max_z, 255));
const int gray_val2 =
@@ -130,11 +136,16 @@ void AddConnectionToRenderData(const LandmarkType& start,
template <class LandmarkListType, class LandmarkType>
void AddConnections(const LandmarkListType& landmarks,
const std::vector<int>& landmark_connections,
bool utilize_visibility, float visibility_threshold,
const Color& connection_color, float thickness,
bool normalized, RenderData* render_data) {
for (int i = 0; i < landmark_connections.size(); i += 2) {
const auto& ld0 = landmarks.landmark(landmark_connections[i]);
const auto& ld1 = landmarks.landmark(landmark_connections[i + 1]);
if (visibility_threshold && (ld0.visibility() < visibility_threshold ||
ld1.visibility() < visibility_threshold)) {
continue;
}
AddConnectionToRenderData<LandmarkType>(ld0, ld1, connection_color,
thickness, normalized, render_data);
}
@@ -231,6 +242,17 @@ REGISTER_CALCULATOR(LandmarksToRenderDataCalculator);
::mediapipe::Status LandmarksToRenderDataCalculator::Process(
CalculatorContext* cc) {
// Check that landmarks are not empty and skip rendering if so.
// Don't emit an empty packet for this timestamp.
if (cc->Inputs().HasTag(kLandmarksTag) &&
cc->Inputs().Tag(kLandmarksTag).IsEmpty()) {
return ::mediapipe::OkStatus();
}
if (cc->Inputs().HasTag(kNormLandmarksTag) &&
cc->Inputs().Tag(kNormLandmarksTag).IsEmpty()) {
return ::mediapipe::OkStatus();
}
auto render_data = absl::make_unique<RenderData>();
bool visualize_depth = options_.visualize_landmark_depth();
float z_min = 0.f;
@@ -255,15 +277,23 @@ REGISTER_CALCULATOR(LandmarksToRenderDataCalculator);
visualize_depth &= ((z_max - z_min) > 1e-3);
if (visualize_depth) {
AddConnectionsWithDepth<LandmarkList, Landmark>(
landmarks, landmark_connections_, thickness, /*normalized=*/false,
landmarks, landmark_connections_, options_.utilize_visibility(),
options_.visibility_threshold(), thickness, /*normalized=*/false,
z_min, z_max, render_data.get());
} else {
AddConnections<LandmarkList, Landmark>(
landmarks, landmark_connections_, options_.connection_color(),
landmarks, landmark_connections_, options_.utilize_visibility(),
options_.visibility_threshold(), options_.connection_color(),
thickness, /*normalized=*/false, render_data.get());
}
for (int i = 0; i < landmarks.landmark_size(); ++i) {
const Landmark& landmark = landmarks.landmark(i);
if (options_.utilize_visibility() &&
landmark.visibility() < options_.visibility_threshold()) {
continue;
}
auto* landmark_data_render = AddPointRenderData(
options_.landmark_color(), thickness, render_data.get());
if (visualize_depth) {
@@ -288,15 +318,23 @@ REGISTER_CALCULATOR(LandmarksToRenderDataCalculator);
visualize_depth &= ((z_max - z_min) > 1e-3);
if (visualize_depth) {
AddConnectionsWithDepth<NormalizedLandmarkList, NormalizedLandmark>(
landmarks, landmark_connections_, thickness, /*normalized=*/true,
landmarks, landmark_connections_, options_.utilize_visibility(),
options_.visibility_threshold(), thickness, /*normalized=*/true,
z_min, z_max, render_data.get());
} else {
AddConnections<NormalizedLandmarkList, NormalizedLandmark>(
landmarks, landmark_connections_, options_.connection_color(),
landmarks, landmark_connections_, options_.utilize_visibility(),
options_.visibility_threshold(), options_.connection_color(),
thickness, /*normalized=*/true, render_data.get());
}
for (int i = 0; i < landmarks.landmark_size(); ++i) {
const NormalizedLandmark& landmark = landmarks.landmark(i);
if (options_.utilize_visibility() &&
landmark.visibility() < options_.visibility_threshold()) {
continue;
}
auto* landmark_data_render = AddPointRenderData(
options_.landmark_color(), thickness, render_data.get());
if (visualize_depth) {
mediapipe/calculators/util/landmarks_to_render_data_calculator.proto
+9
View File
@@ -40,4 +40,13 @@ message LandmarksToRenderDataCalculatorOptions {
// Change color and size of rendered landmarks based on its z value.
optional bool visualize_landmark_depth = 5 [default = true];
// Use landmarks visibility while rendering landmarks and connections. If
// landmark is not visible, neither it nor adjacent connections will be
// rendered.
optional bool utilize_visibility = 6 [default = false];
// Threshold to determine visibility of the landmark. Landmark with visibility
// greater or equal than threshold is considered visible.
optional double visibility_threshold = 7 [default = 0.0];
}
mediapipe/calculators/util/rect_projection_calculator.cc
+100
View File
@@ -0,0 +1,100 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <cmath>
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/rect.pb.h"
namespace mediapipe {
namespace {
constexpr char kNormRectTag[] = "NORM_RECT";
constexpr char kNormReferenceRectTag[] = "NORM_REFERENCE_RECT";
} // namespace
// Projects rectangle from reference coordinate system (defined by reference
// rectangle) to original coordinate system (in which this reference rectangle
// is defined).
//
// Inputs:
// NORM_RECT - A NormalizedRect to be projected.
// NORM_REFERENCE_RECT - A NormalizedRect that represents reference coordinate
// system for NORM_RECT and is defined in original coordinates.
//
// Outputs:
// NORM_RECT: A NormalizedRect projected to the original coordinates.
//
// Example config:
// node {
// calculator: "RectProjectionCalculator"
// input_stream: "NORM_RECT:face_rect"
// input_stream: "NORM_REFERENCE_RECT:face_reference_rect"
// output_stream: "NORM_RECT:projected_face_rect"
// }
//
class RectProjectionCalculator : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc);
::mediapipe::Status Process(CalculatorContext* cc) override;
};
REGISTER_CALCULATOR(RectProjectionCalculator);
::mediapipe::Status RectProjectionCalculator::GetContract(
CalculatorContract* cc) {
cc->Inputs().Tag(kNormRectTag).Set<NormalizedRect>();
cc->Inputs().Tag(kNormReferenceRectTag).Set<NormalizedRect>();
cc->Outputs().Tag(kNormRectTag).Set<NormalizedRect>();
return ::mediapipe::OkStatus();
}
::mediapipe::Status RectProjectionCalculator::Process(CalculatorContext* cc) {
if (cc->Inputs().Tag(kNormRectTag).IsEmpty()) {
return ::mediapipe::OkStatus();
}
const auto& rect = cc->Inputs().Tag(kNormRectTag).Get<NormalizedRect>();
const auto& reference_rect =
cc->Inputs().Tag(kNormReferenceRectTag).Get<NormalizedRect>();
// Project center.
const float x = rect.x_center() - 0.5f;
const float y = rect.y_center() - 0.5f;
const float angle = reference_rect.rotation();
float new_x = std::cos(angle) * x - std::sin(angle) * y;
float new_y = std::sin(angle) * x + std::cos(angle) * y;
new_x = new_x * reference_rect.width() + reference_rect.x_center();
new_y = new_y * reference_rect.height() + reference_rect.y_center();
// Project size.
const float new_width = rect.width() * reference_rect.width();
const float new_height = rect.height() * reference_rect.height();
// Project rotation.
const float new_rotation = rect.rotation() + reference_rect.rotation();
auto new_rect = absl::make_unique<NormalizedRect>();
new_rect->set_x_center(new_x);
new_rect->set_y_center(new_y);
new_rect->set_width(new_width);
new_rect->set_height(new_height);
new_rect->set_rotation(new_rotation);
cc->Outputs().Tag(kNormRectTag).Add(new_rect.release(), cc->InputTimestamp());
return ::mediapipe::OkStatus();
}
} // namespace mediapipe
mediapipe/calculators/util/top_k_scores_calculator.cc
+2 -1
View File
@@ -21,6 +21,7 @@
#include <utility>
#include <vector>
#include "absl/container/node_hash_map.h"
#include "mediapipe/calculators/util/top_k_scores_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/classification.pb.h"
@@ -72,7 +73,7 @@ class TopKScoresCalculator : public CalculatorBase {
int top_k_ = -1;
float threshold_ = 0.0;
std::unordered_map<int, std::string> label_map_;
absl::node_hash_map<int, std::string> label_map_;
bool label_map_loaded_ = false;
};
REGISTER_CALCULATOR(TopKScoresCalculator);