Project import generated by Copybara.

GitOrigin-RevId: 283c1a295de0a53e47d7a94996bda0c52dcfd677
This commit is contained in:
MediaPipe Team
2021-09-13 21:35:51 -04:00
committed by chuoling
parent 6abec128ed
commit 137e1cc763
31 changed files with 2051 additions and 53 deletions
+51
View File
@@ -356,6 +356,57 @@ cc_library(
alwayslink = 1,
)
mediapipe_proto_library(
name = "landmarks_to_tensor_calculator_proto",
srcs = ["landmarks_to_tensor_calculator.proto"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
cc_library(
name = "landmarks_to_tensor_calculator",
srcs = ["landmarks_to_tensor_calculator.cc"],
hdrs = ["landmarks_to_tensor_calculator.h"],
copts = select({
"//mediapipe:apple": [
"-x objective-c++",
"-fobjc-arc", # enable reference-counting
],
"//conditions:default": [],
}),
visibility = ["//visibility:public"],
deps = [
":landmarks_to_tensor_calculator_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/api2:node",
"//mediapipe/framework/formats:landmark_cc_proto",
"//mediapipe/framework/formats:tensor",
"//mediapipe/framework/port:ret_check",
],
alwayslink = 1,
)
cc_test(
name = "landmarks_to_tensor_calculator_test",
srcs = ["landmarks_to_tensor_calculator_test.cc"],
deps = [
":landmarks_to_tensor_calculator",
":landmarks_to_tensor_calculator_cc_proto",
"//mediapipe/framework:calculator_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:calculator_runner",
"//mediapipe/framework/formats:landmark_cc_proto",
"//mediapipe/framework/formats:tensor",
"//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:parse_text_proto",
"@com_google_absl//absl/memory",
"@com_google_googletest//:gtest_main",
],
)
mediapipe_proto_library(
name = "tensors_to_floats_calculator_proto",
srcs = ["tensors_to_floats_calculator.proto"],
@@ -99,13 +99,11 @@ class InferenceCalculator : public NodeIntf {
kSideInCustomOpResolver{"CUSTOM_OP_RESOLVER"};
static constexpr SideInput<TfLiteModelPtr>::Optional kSideInModel{"MODEL"};
static constexpr Output<std::vector<Tensor>> kOutTensors{"TENSORS"};
static constexpr SideInput<std::string>::Optional kNnApiDelegateCacheDir{
"NNAPI_CACHE_DIR"};
static constexpr SideInput<std::string>::Optional kNnApiDelegateModelToken{
"NNAPI_MODEL_TOKEN"};
static constexpr SideInput<
mediapipe::InferenceCalculatorOptions::Delegate>::Optional kDelegate{
"DELEGATE"};
MEDIAPIPE_NODE_CONTRACT(kInTensors, kSideInCustomOpResolver, kSideInModel,
kOutTensors, kNnApiDelegateCacheDir,
kNnApiDelegateModelToken);
kOutTensors, kDelegate);
protected:
using TfLiteDelegatePtr =
@@ -18,6 +18,9 @@ package mediapipe;
import "mediapipe/framework/calculator.proto";
option java_package = "com.google.mediapipe.calculator.proto";
option java_outer_classname = "InferenceCalculatorProto";
// Full Example:
//
// node {
@@ -50,11 +50,13 @@ int GetXnnpackDefaultNumThreads() {
// Returns number of threads to configure XNNPACK delegate with.
// Returns user provided value if specified. Otherwise, tries to choose optimal
// number of threads depending on the device.
int GetXnnpackNumThreads(const mediapipe::InferenceCalculatorOptions& opts) {
int GetXnnpackNumThreads(
const bool opts_has_delegate,
const mediapipe::InferenceCalculatorOptions::Delegate& opts_delegate) {
static constexpr int kDefaultNumThreads = -1;
if (opts.has_delegate() && opts.delegate().has_xnnpack() &&
opts.delegate().xnnpack().num_threads() != kDefaultNumThreads) {
return opts.delegate().xnnpack().num_threads();
if (opts_has_delegate && opts_delegate.has_xnnpack() &&
opts_delegate.xnnpack().num_threads() != kDefaultNumThreads) {
return opts_delegate.xnnpack().num_threads();
}
return GetXnnpackDefaultNumThreads();
}
@@ -175,33 +177,40 @@ absl::Status InferenceCalculatorCpuImpl::LoadDelegateAndAllocateTensors(
absl::Status InferenceCalculatorCpuImpl::LoadDelegate(CalculatorContext* cc) {
const auto& calculator_opts =
cc->Options<mediapipe::InferenceCalculatorOptions>();
if (calculator_opts.has_delegate() &&
calculator_opts.delegate().has_tflite()) {
auto opts_delegate = calculator_opts.delegate();
if (!kDelegate(cc).IsEmpty()) {
mediapipe::InferenceCalculatorOptions::Delegate input_side_packet_delegate =
kDelegate(cc).Get();
CHECK(input_side_packet_delegate.has_tflite() ||
input_side_packet_delegate.has_xnnpack() ||
input_side_packet_delegate.has_nnapi() ||
input_side_packet_delegate.delegate_case() ==
mediapipe::InferenceCalculatorOptions::Delegate::DELEGATE_NOT_SET)
<< "inference_calculator_cpu only supports delegate input side packet "
<< "for TFLite, XNNPack and Nnapi";
opts_delegate.MergeFrom(input_side_packet_delegate);
}
const bool opts_has_delegate =
calculator_opts.has_delegate() || !kDelegate(cc).IsEmpty();
if (opts_has_delegate && opts_delegate.has_tflite()) {
// Default tflite inference requeqsted - no need to modify graph.
return absl::OkStatus();
}
#if defined(MEDIAPIPE_ANDROID)
const bool nnapi_requested = calculator_opts.has_delegate()
? calculator_opts.delegate().has_nnapi()
: calculator_opts.use_nnapi();
const bool nnapi_requested = opts_has_delegate ? opts_delegate.has_nnapi()
: calculator_opts.use_nnapi();
if (nnapi_requested) {
// Attempt to use NNAPI.
// If not supported, the default CPU delegate will be created and used.
interpreter_->SetAllowFp16PrecisionForFp32(1);
tflite::StatefulNnApiDelegate::Options options;
const auto& nnapi = calculator_opts.delegate().nnapi();
const auto& nnapi = opts_delegate.nnapi();
// Set up cache_dir and model_token for NNAPI compilation cache.
options.cache_dir =
nnapi.has_cache_dir() ? nnapi.cache_dir().c_str() : nullptr;
if (!kNnApiDelegateCacheDir(cc).IsEmpty()) {
options.cache_dir = kNnApiDelegateCacheDir(cc).Get().c_str();
}
options.model_token =
nnapi.has_model_token() ? nnapi.model_token().c_str() : nullptr;
if (!kNnApiDelegateModelToken(cc).IsEmpty()) {
options.model_token = kNnApiDelegateModelToken(cc).Get().c_str();
}
delegate_ = TfLiteDelegatePtr(new tflite::StatefulNnApiDelegate(options),
[](TfLiteDelegate*) {});
RET_CHECK_EQ(interpreter_->ModifyGraphWithDelegate(delegate_.get()),
@@ -213,13 +222,13 @@ absl::Status InferenceCalculatorCpuImpl::LoadDelegate(CalculatorContext* cc) {
#if defined(__EMSCRIPTEN__)
const bool use_xnnpack = true;
#else
const bool use_xnnpack = calculator_opts.has_delegate() &&
calculator_opts.delegate().has_xnnpack();
const bool use_xnnpack = opts_has_delegate && opts_delegate.has_xnnpack();
#endif // defined(__EMSCRIPTEN__)
if (use_xnnpack) {
TfLiteXNNPackDelegateOptions xnnpack_opts{};
xnnpack_opts.num_threads = GetXnnpackNumThreads(calculator_opts);
xnnpack_opts.num_threads =
GetXnnpackNumThreads(opts_has_delegate, opts_delegate);
delegate_ = TfLiteDelegatePtr(TfLiteXNNPackDelegateCreate(&xnnpack_opts),
&TfLiteXNNPackDelegateDelete);
RET_CHECK_EQ(interpreter_->ModifyGraphWithDelegate(delegate_.get()),
@@ -95,19 +95,30 @@ absl::Status InferenceCalculatorGlImpl::UpdateContract(CalculatorContract* cc) {
absl::Status InferenceCalculatorGlImpl::Open(CalculatorContext* cc) {
const auto& options = cc->Options<::mediapipe::InferenceCalculatorOptions>();
use_advanced_gpu_api_ = options.has_delegate() &&
options.delegate().has_gpu() &&
options.delegate().gpu().use_advanced_gpu_api();
allow_precision_loss_ = options.delegate().gpu().allow_precision_loss();
tflite_gpu_runner_api_ = options.delegate().gpu().api();
tflite_gpu_runner_usage_ = options.delegate().gpu().usage();
use_kernel_caching_ = use_advanced_gpu_api_ &&
options.delegate().gpu().has_cached_kernel_path();
mediapipe::InferenceCalculatorOptions::Delegate delegate = options.delegate();
if (!kDelegate(cc).IsEmpty()) {
mediapipe::InferenceCalculatorOptions::Delegate input_side_packet_delegate =
kDelegate(cc).Get();
CHECK(input_side_packet_delegate.has_gpu() ||
input_side_packet_delegate.delegate_case() ==
mediapipe::InferenceCalculatorOptions::Delegate::DELEGATE_NOT_SET)
<< "inference_calculator_gl only supports delegate input side packet "
<< "for Gpu";
delegate.MergeFrom(input_side_packet_delegate);
}
const bool has_delegate = options.has_delegate() || !kDelegate(cc).IsEmpty();
use_advanced_gpu_api_ = has_delegate && delegate.has_gpu() &&
delegate.gpu().use_advanced_gpu_api();
allow_precision_loss_ = delegate.gpu().allow_precision_loss();
tflite_gpu_runner_api_ = delegate.gpu().api();
tflite_gpu_runner_usage_ = delegate.gpu().usage();
use_kernel_caching_ =
use_advanced_gpu_api_ && delegate.gpu().has_cached_kernel_path();
use_gpu_delegate_ = !use_advanced_gpu_api_;
if (use_kernel_caching_) {
#ifdef MEDIAPIPE_ANDROID
cached_kernel_filename_ = options.delegate().gpu().cached_kernel_path() +
cached_kernel_filename_ = delegate.gpu().cached_kernel_path() +
mediapipe::File::Basename(options.model_path()) +
".ker";
#endif // MEDIAPIPE_ANDROID
@@ -0,0 +1,101 @@
// Copyright 2021 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/calculators/tensor/landmarks_to_tensor_calculator.h"
#include <memory>
#include "mediapipe/calculators/tensor/landmarks_to_tensor_calculator.pb.h"
#include "mediapipe/framework/api2/node.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/landmark.pb.h"
#include "mediapipe/framework/formats/tensor.h"
#include "mediapipe/framework/port/ret_check.h"
namespace mediapipe {
namespace api2 {
namespace {
float GetAttribute(
const Landmark& landmark,
const LandmarksToTensorCalculatorOptions::Attribute& attribute) {
switch (attribute) {
case LandmarksToTensorCalculatorOptions::X:
return landmark.x();
case LandmarksToTensorCalculatorOptions::Y:
return landmark.y();
case LandmarksToTensorCalculatorOptions::Z:
return landmark.z();
case LandmarksToTensorCalculatorOptions::VISIBILITY:
return landmark.visibility();
case LandmarksToTensorCalculatorOptions::PRESENCE:
return landmark.presence();
}
}
} // namespace
class LandmarksToTensorCalculatorImpl
: public NodeImpl<LandmarksToTensorCalculator> {
public:
absl::Status Open(CalculatorContext* cc) override {
options_ = cc->Options<LandmarksToTensorCalculatorOptions>();
RET_CHECK(options_.attributes_size() > 0)
<< "At least one attribute must be specified";
return absl::OkStatus();
}
absl::Status Process(CalculatorContext* cc) override {
if (kInLandmarkList(cc).IsEmpty()) {
return absl::OkStatus();
}
// Get input landmarks.
const auto& in_landmarks = *kInLandmarkList(cc);
// Determine tensor shape.
const int n_landmarks = in_landmarks.landmark_size();
const int n_attributes = options_.attributes_size();
auto tensor_shape = options_.flatten()
? Tensor::Shape{1, n_landmarks * n_attributes}
: Tensor::Shape{1, n_landmarks, n_attributes};
// Create empty tesnor.
Tensor tensor(Tensor::ElementType::kFloat32, tensor_shape);
auto* buffer = tensor.GetCpuWriteView().buffer<float>();
// Fill tensor with landmark attributes.
for (int i = 0; i < n_landmarks; ++i) {
for (int j = 0; j < n_attributes; ++j) {
buffer[i * n_attributes + j] =
GetAttribute(in_landmarks.landmark(i), options_.attributes(j));
}
}
// Return vector with a single tensor.
auto result = std::vector<Tensor>();
result.push_back(std::move(tensor));
kOutTensors(cc).Send(std::move(result));
return absl::OkStatus();
}
private:
LandmarksToTensorCalculatorOptions options_;
};
MEDIAPIPE_NODE_IMPLEMENTATION(LandmarksToTensorCalculatorImpl);
} // namespace api2
} // namespace mediapipe
@@ -0,0 +1,61 @@
// Copyright 2021 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef MEDIAPIPE_CALCULATORS_LANDMARKS_TO_TENSOR_CALCULATOR_H_
#define MEDIAPIPE_CALCULATORS_LANDMARKS_TO_TENSOR_CALCULATOR_H_
#include <memory>
#include "mediapipe/framework/api2/node.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/landmark.pb.h"
#include "mediapipe/framework/formats/tensor.h"
namespace mediapipe {
namespace api2 {
// A calculator for converting landmars into a Tensor.
//
// Input:
// LANDMARKS - LandmarkList
// Landmarks to be converted into a Tensor.
//
// Output:
// TENSORS - std::vector<Tensor>
// Vector containing a single Tensor populated with landmark values.
//
// Example:
// node {
// calculator: "LandmarksToTensorCalculator"
// input_stream: "LANDMARKS:landmarks"
// output_stream: "TENSORS:tensors"
// options: {
// [mediapipe.LandmarksToTensorCalculatorOptions.ext] {
// attributes: [X, Y, Z, VISIBILITY, PRESENCE]
// # flatten: true
// }
// }
// }
class LandmarksToTensorCalculator : public NodeIntf {
public:
static constexpr Input<LandmarkList>::Optional kInLandmarkList{"LANDMARKS"};
static constexpr Output<std::vector<Tensor>> kOutTensors{"TENSORS"};
MEDIAPIPE_NODE_INTERFACE(LandmarksToTensorCalculator, kInLandmarkList,
kOutTensors);
};
} // namespace api2
} // namespace mediapipe
#endif // MEDIAPIPE_CALCULATORS_LANDMARKS_TO_TENSOR_CALCULATOR_H_
@@ -0,0 +1,44 @@
// Copyright 2021 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// The option proto for the LandmarksToTensorCalculator.
syntax = "proto2";
package mediapipe;
import "mediapipe/framework/calculator.proto";
message LandmarksToTensorCalculatorOptions {
extend mediapipe.CalculatorOptions {
optional LandmarksToTensorCalculatorOptions ext = 394810235;
}
enum Attribute {
X = 0;
Y = 1;
Z = 2;
VISIBILITY = 3;
PRESENCE = 4;
}
// Subset and order of attributes as they should appear in the output Tensor.
// Should contain at least one attribute.
repeated Attribute attributes = 1;
// Collapses all landmark attributes into a one dimensional tensor (i.e.
// switches from (n_landmarks, n_attributes) to (n_landmarks * n_attributes)
// representation).
optional bool flatten = 2 [default = false];
}
@@ -0,0 +1,155 @@
// Copyright 2021 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <vector>
#include "absl/memory/memory.h"
#include "mediapipe/calculators/tensor/landmarks_to_tensor_calculator.pb.h"
#include "mediapipe/framework/calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_runner.h"
#include "mediapipe/framework/formats/landmark.pb.h"
#include "mediapipe/framework/formats/tensor.h"
#include "mediapipe/framework/port/gmock.h"
#include "mediapipe/framework/port/gtest.h"
#include "mediapipe/framework/port/parse_text_proto.h"
#include "mediapipe/framework/port/status_matchers.h"
namespace mediapipe {
namespace {
using ::mediapipe::ParseTextProtoOrDie;
using Node = ::mediapipe::CalculatorGraphConfig::Node;
void RunLandmarks(mediapipe::CalculatorRunner* runner,
const LandmarkList& landmarks) {
runner->MutableInputs()
->Tag("LANDMARKS")
.packets.push_back(MakePacket<LandmarkList>(landmarks).At(Timestamp(0)));
MP_ASSERT_OK(runner->Run());
}
const Tensor& GetOutputTensor(mediapipe::CalculatorRunner* runner) {
const auto& output_packets = runner->Outputs().Tag("TENSORS").packets;
EXPECT_EQ(output_packets.size(), 1);
const auto& tensors = output_packets[0].Get<std::vector<Tensor>>();
EXPECT_EQ(tensors.size(), 1);
return tensors[0];
}
void ValidateTensor(const Tensor& tensor,
const std::vector<int>& expected_shape,
const std::vector<float>& expected_values) {
EXPECT_EQ(tensor.shape().dims, expected_shape);
EXPECT_EQ(tensor.shape().num_elements(), expected_values.size());
auto* tensor_buffer = tensor.GetCpuReadView().buffer<float>();
const std::vector<float> tensor_values(
tensor_buffer, tensor_buffer + tensor.shape().num_elements());
EXPECT_THAT(tensor_values, testing::ElementsAreArray(expected_values));
}
TEST(LandmarksToTensorCalculatorTest, AllAttributes) {
mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"pb(
calculator: "LandmarksToTensorCalculator"
input_stream: "LANDMARKS:landmarks"
output_stream: "TENSORS:tensors"
options: {
[mediapipe.LandmarksToTensorCalculatorOptions.ext] {
attributes: [ X, Y, Z, VISIBILITY, PRESENCE ]
}
}
)pb"));
LandmarkList landmarks;
auto* landmark1 = landmarks.add_landmark();
landmark1->set_x(1.0f);
landmark1->set_y(2.0f);
landmark1->set_z(3.0f);
landmark1->set_visibility(4.0f);
landmark1->set_presence(5.0f);
auto* landmark2 = landmarks.add_landmark();
landmark2->set_x(6.0f);
landmark2->set_y(7.0f);
landmark2->set_z(8.0f);
landmark2->set_visibility(9.0f);
landmark2->set_presence(10.0f);
RunLandmarks(&runner, landmarks);
const auto& tensor = GetOutputTensor(&runner);
ValidateTensor(tensor, /*expected_shape=*/{1, 2, 5}, /*expected_values=*/
{1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f, 9.0f, 10.0f});
}
TEST(LandmarksToTensorCalculatorTest, XYZAttributes) {
mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"pb(
calculator: "LandmarksToTensorCalculator"
input_stream: "LANDMARKS:landmarks"
output_stream: "TENSORS:tensors"
options: {
[mediapipe.LandmarksToTensorCalculatorOptions.ext] {
attributes: [ X, Y, Z ]
}
}
)pb"));
LandmarkList landmarks;
auto* landmark1 = landmarks.add_landmark();
landmark1->set_x(1.0f);
landmark1->set_y(2.0f);
landmark1->set_z(3.0f);
auto* landmark2 = landmarks.add_landmark();
landmark2->set_x(6.0f);
landmark2->set_y(7.0f);
landmark2->set_z(8.0f);
RunLandmarks(&runner, landmarks);
const auto& tensor = GetOutputTensor(&runner);
ValidateTensor(tensor, /*expected_shape=*/{1, 2, 3}, /*expected_values=*/
{1.0f, 2.0f, 3.0f, 6.0f, 7.0f, 8.0f});
}
TEST(LandmarksToTensorCalculatorTest, XYZAttributes_Flatten) {
mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"pb(
calculator: "LandmarksToTensorCalculator"
input_stream: "LANDMARKS:landmarks"
output_stream: "TENSORS:tensors"
options: {
[mediapipe.LandmarksToTensorCalculatorOptions.ext] {
attributes: [ X, Y, Z ]
flatten: true
}
}
)pb"));
LandmarkList landmarks;
auto* landmark1 = landmarks.add_landmark();
landmark1->set_x(1.0f);
landmark1->set_y(2.0f);
landmark1->set_z(3.0f);
auto* landmark2 = landmarks.add_landmark();
landmark2->set_x(6.0f);
landmark2->set_y(7.0f);
landmark2->set_z(8.0f);
RunLandmarks(&runner, landmarks);
const auto& tensor = GetOutputTensor(&runner);
ValidateTensor(tensor, /*expected_shape=*/{1, 6}, /*expected_values=*/
{1.0f, 2.0f, 3.0f, 6.0f, 7.0f, 8.0f});
}
} // namespace
} // namespace mediapipe