talksik/mediapipe
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Project import generated by Copybara.

Browse Source 
GitOrigin-RevId: 6f964e58d874e47fb6207aa97d060a4cd6428527
This commit is contained in:
MediaPipe Team
2020-03-02 10:35:07 -08:00
committed by jqtang
parent de4fbc10e6
commit 252a5713c7
134 changed files with 16665 additions and 874 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mediapipe/calculators/core/BUILD
+16
View File
@@ -610,6 +610,22 @@ cc_library(
alwayslink = 1,
)
cc_test(
name = "side_packet_to_stream_calculator_test",
srcs = ["side_packet_to_stream_calculator_test.cc"],
deps = [
":side_packet_to_stream_calculator",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:integral_types",
"//mediapipe/framework/port:parse_text_proto",
"//mediapipe/framework/port:status",
"//mediapipe/framework/tool:options_util",
"@com_google_absl//absl/memory",
"@com_google_absl//absl/strings",
],
)
cc_test(
name = "immediate_mux_calculator_test",
srcs = ["immediate_mux_calculator_test.cc"],
mediapipe/calculators/core/side_packet_to_stream_calculator.cc
+96 -18
View File
@@ -28,55 +28,133 @@ using mediapipe::PacketTypeSet;
using mediapipe::Timestamp;
namespace {
constexpr char kTagAtPreStream[] = "AT_PRESTREAM";
constexpr char kTagAtPostStream[] = "AT_POSTSTREAM";
constexpr char kTagAtZero[] = "AT_ZERO";
constexpr char kTagAtTick[] = "AT_TICK";
constexpr char kTagTick[] = "TICK";
static std::map<std::string, Timestamp>* kTimestampMap = []() {
auto* res = new std::map<std::string, Timestamp>();
res->emplace("AT_PRESTREAM", Timestamp::PreStream());
res->emplace("AT_POSTSTREAM", Timestamp::PostStream());
res->emplace("AT_ZERO", Timestamp(0));
res->emplace(kTagAtPreStream, Timestamp::PreStream());
res->emplace(kTagAtPostStream, Timestamp::PostStream());
res->emplace(kTagAtZero, Timestamp(0));
res->emplace(kTagAtTick, Timestamp::Unset());
return res;
}();
template <typename CC>
std::string GetOutputTag(const CC& cc) {
// Single output tag only is required by contract.
return *cc.Outputs().GetTags().begin();
}
} // namespace
// Outputs the single input_side_packet at the timestamp specified in the
// output_stream tag. Valid tags are AT_PRESTREAM, AT_POSTSTREAM and AT_ZERO.
// Outputs side packet(s) in corresponding output stream(s) with a particular
// timestamp, depending on the tag used to define output stream(s). (One tag can
// be used only.)
//
// Valid tags are AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO and AT_TICK and
// corresponding timestamps are Timestamp::PreStream(), Timestamp::PostStream(),
// Timestamp(0) and timestamp of a packet received in TICK input.
//
// Examples:
// node {
// calculator: "SidePacketToStreamCalculator"
// input_side_packet: "side_packet"
// output_stream: "AT_PRESTREAM:packet"
// }
//
// node {
// calculator: "SidePacketToStreamCalculator"
// input_stream: "TICK:tick"
// input_side_packet: "side_packet"
// output_stream: "AT_TICK:packet"
// }
class SidePacketToStreamCalculator : public CalculatorBase {
public:
SidePacketToStreamCalculator() = default;
~SidePacketToStreamCalculator() override = default;
static ::mediapipe::Status GetContract(CalculatorContract* cc);
::mediapipe::Status Open(CalculatorContext* cc) override;
::mediapipe::Status Process(CalculatorContext* cc) override;
::mediapipe::Status Close(CalculatorContext* cc) override;
private:
bool is_tick_processing_ = false;
std::string output_tag_;
};
REGISTER_CALCULATOR(SidePacketToStreamCalculator);
::mediapipe::Status SidePacketToStreamCalculator::GetContract(
CalculatorContract* cc) {
cc->InputSidePackets().Index(0).SetAny();
const auto& tags = cc->Outputs().GetTags();
RET_CHECK(tags.size() == 1 && kTimestampMap->count(*tags.begin()) == 1)
<< "Only one of AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO and AT_TICK tags is "
"allowed and required to specify output stream(s).";
RET_CHECK(
(cc->Outputs().HasTag(kTagAtTick) && cc->Inputs().HasTag(kTagTick)) ||
(!cc->Outputs().HasTag(kTagAtTick) && !cc->Inputs().HasTag(kTagTick)))
<< "Either both of TICK and AT_TICK should be used or none of them.";
const std::string output_tag = GetOutputTag(*cc);
const int num_entries = cc->Outputs().NumEntries(output_tag);
RET_CHECK_EQ(num_entries, cc->InputSidePackets().NumEntries())
<< "Same number of input side packets and output streams is required.";
for (int i = 0; i < num_entries; ++i) {
cc->InputSidePackets().Index(i).SetAny();
cc->Outputs()
.Get(output_tag, i)
.SetSameAs(cc->InputSidePackets().Index(i).GetSameAs());
}
std::set<std::string> tags = cc->Outputs().GetTags();
RET_CHECK_EQ(tags.size(), 1);
if (cc->Inputs().HasTag(kTagTick)) {
cc->Inputs().Tag(kTagTick).SetAny();
}
RET_CHECK_EQ(kTimestampMap->count(*tags.begin()), 1);
cc->Outputs().Tag(*tags.begin()).SetAny();
return ::mediapipe::OkStatus();
}
::mediapipe::Status SidePacketToStreamCalculator::Open(CalculatorContext* cc) {
output_tag_ = GetOutputTag(*cc);
if (cc->Inputs().HasTag(kTagTick)) {
is_tick_processing_ = true;
// Set offset, so output timestamp bounds are updated in response to TICK
// timestamp bound update.
cc->SetOffset(TimestampDiff(0));
}
return ::mediapipe::OkStatus();
}
::mediapipe::Status SidePacketToStreamCalculator::Process(
CalculatorContext* cc) {
return mediapipe::tool::StatusStop();
if (is_tick_processing_) {
// TICK input is guaranteed to be non-empty, as it's the only input stream
// for this calculator.
const auto& timestamp = cc->Inputs().Tag(kTagTick).Value().Timestamp();
for (int i = 0; i < cc->Outputs().NumEntries(output_tag_); ++i) {
cc->Outputs()
.Get(output_tag_, i)
.AddPacket(cc->InputSidePackets().Index(i).At(timestamp));
}
return ::mediapipe::OkStatus();
}
return ::mediapipe::tool::StatusStop();
}
::mediapipe::Status SidePacketToStreamCalculator::Close(CalculatorContext* cc) {
std::set<std::string> tags = cc->Outputs().GetTags();
RET_CHECK_EQ(tags.size(), 1);
const std::string& tag = *tags.begin();
RET_CHECK_EQ(kTimestampMap->count(tag), 1);
cc->Outputs().Tag(tag).AddPacket(
cc->InputSidePackets().Index(0).At(kTimestampMap->at(tag)));
if (!cc->Outputs().HasTag(kTagAtTick)) {
const auto& timestamp = kTimestampMap->at(output_tag_);
for (int i = 0; i < cc->Outputs().NumEntries(output_tag_); ++i) {
cc->Outputs()
.Get(output_tag_, i)
.AddPacket(cc->InputSidePackets().Index(i).At(timestamp));
}
}
return ::mediapipe::OkStatus();
}
mediapipe/calculators/core/side_packet_to_stream_calculator_test.cc
+275
View File
@@ -0,0 +1,275 @@
// Copyright 2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <string>
#include <vector>
#include "absl/memory/memory.h"
#include "absl/strings/match.h"
#include "absl/strings/str_replace.h"
#include "absl/strings/string_view.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/port/gtest.h"
#include "mediapipe/framework/port/integral_types.h"
#include "mediapipe/framework/port/parse_text_proto.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/framework/port/status_matchers.h"
#include "mediapipe/framework/tool/options_util.h"
namespace mediapipe {
namespace {
TEST(SidePacketToStreamCalculator, WrongConfig_MissingTick) {
CalculatorGraphConfig graph_config =
ParseTextProtoOrDie<CalculatorGraphConfig>(
R"(
input_stream: "tick"
input_side_packet: "side_packet"
output_stream: "packet"
node {
calculator: "SidePacketToStreamCalculator"
input_side_packet: "side_packet"
output_stream: "AT_TICK:packet"
}
)");
CalculatorGraph graph;
auto status = graph.Initialize(graph_config);
EXPECT_FALSE(status.ok());
EXPECT_PRED2(
absl::StrContains, status.message(),
"Either both of TICK and AT_TICK should be used or none of them.");
}
TEST(SidePacketToStreamCalculator, WrongConfig_NonExistentTag) {
CalculatorGraphConfig graph_config =
ParseTextProtoOrDie<CalculatorGraphConfig>(
R"(
input_stream: "tick"
input_side_packet: "side_packet"
output_stream: "packet"
node {
calculator: "SidePacketToStreamCalculator"
input_side_packet: "side_packet"
output_stream: "DOES_NOT_EXIST:packet"
}
)");
CalculatorGraph graph;
auto status = graph.Initialize(graph_config);
EXPECT_FALSE(status.ok());
EXPECT_PRED2(absl::StrContains, status.message(),
"Only one of AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO and AT_TICK "
"tags is allowed and required to specify output stream(s).");
}
TEST(SidePacketToStreamCalculator, WrongConfig_MixedTags) {
CalculatorGraphConfig graph_config =
ParseTextProtoOrDie<CalculatorGraphConfig>(
R"(
input_stream: "tick"
input_side_packet: "side_packet0"
input_side_packet: "side_packet1"
node {
calculator: "SidePacketToStreamCalculator"
input_side_packet: "side_packet0"
input_side_packet: "side_packet1"
output_stream: "AT_TICK:packet0"
output_stream: "AT_PRE_STREAM:packet1"
}
)");
CalculatorGraph graph;
auto status = graph.Initialize(graph_config);
EXPECT_FALSE(status.ok());
EXPECT_PRED2(absl::StrContains, status.message(),
"Only one of AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO and AT_TICK "
"tags is allowed and required to specify output stream(s).");
}
TEST(SidePacketToStreamCalculator, WrongConfig_NotEnoughSidePackets) {
CalculatorGraphConfig graph_config =
ParseTextProtoOrDie<CalculatorGraphConfig>(
R"(
input_side_packet: "side_packet0"
input_side_packet: "side_packet1"
node {
calculator: "SidePacketToStreamCalculator"
input_side_packet: "side_packet0"
output_stream: "AT_PRESTREAM:0:packet0"
output_stream: "AT_PRESTREAM:1:packet1"
}
)");
CalculatorGraph graph;
auto status = graph.Initialize(graph_config);
EXPECT_FALSE(status.ok());
EXPECT_PRED2(
absl::StrContains, status.message(),
"Same number of input side packets and output streams is required.");
}
TEST(SidePacketToStreamCalculator, WrongConfig_NotEnoughOutputStreams) {
CalculatorGraphConfig graph_config =
ParseTextProtoOrDie<CalculatorGraphConfig>(
R"(
input_side_packet: "side_packet0"
input_side_packet: "side_packet1"
node {
calculator: "SidePacketToStreamCalculator"
input_side_packet: "side_packet0"
input_side_packet: "side_packet1"
output_stream: "AT_PRESTREAM:packet0"
}
)");
CalculatorGraph graph;
auto status = graph.Initialize(graph_config);
EXPECT_FALSE(status.ok());
EXPECT_PRED2(
absl::StrContains, status.message(),
"Same number of input side packets and output streams is required.");
}
void DoTestNonAtTickOutputTag(absl::string_view tag,
Timestamp expected_timestamp) {
CalculatorGraphConfig graph_config =
ParseTextProtoOrDie<CalculatorGraphConfig>(absl::StrReplaceAll(
R"(
input_side_packet: "side_packet"
output_stream: "packet"
node {
calculator: "SidePacketToStreamCalculator"
input_side_packet: "side_packet"
output_stream: "$tag:packet"
}
)",
{{"$tag", tag}}));
CalculatorGraph graph;
MP_ASSERT_OK(graph.Initialize(graph_config));
const int expected_value = 10;
std::vector<Packet> output_packets;
MP_ASSERT_OK(graph.ObserveOutputStream(
"packet", [&output_packets](const Packet& packet) {
output_packets.push_back(packet);
return ::mediapipe::OkStatus();
}));
MP_ASSERT_OK(
graph.StartRun({{"side_packet", MakePacket<int>(expected_value)}}));
MP_ASSERT_OK(graph.WaitForObservedOutput());
ASSERT_FALSE(output_packets.empty());
EXPECT_EQ(expected_timestamp, output_packets.back().Timestamp());
EXPECT_EQ(expected_value, output_packets.back().Get<int>());
}
TEST(SidePacketToStreamCalculator, NoAtTickOutputTags) {
DoTestNonAtTickOutputTag("AT_PRESTREAM", Timestamp::PreStream());
DoTestNonAtTickOutputTag("AT_POSTSTREAM", Timestamp::PostStream());
DoTestNonAtTickOutputTag("AT_ZERO", Timestamp(0));
}
TEST(SidePacketToStreamCalculator, AtTick) {
CalculatorGraphConfig graph_config =
ParseTextProtoOrDie<CalculatorGraphConfig>(
R"(
input_stream: "tick"
input_side_packet: "side_packet"
output_stream: "packet"
node {
calculator: "SidePacketToStreamCalculator"
input_stream: "TICK:tick"
input_side_packet: "side_packet"
output_stream: "AT_TICK:packet"
}
)");
std::vector<Packet> output_packets;
tool::AddVectorSink("packet", &graph_config, &output_packets);
CalculatorGraph graph;
MP_ASSERT_OK(graph.Initialize(graph_config));
const int expected_value = 20;
MP_ASSERT_OK(
graph.StartRun({{"side_packet", MakePacket<int>(expected_value)}}));
auto tick_and_verify = [&graph, &output_packets,
expected_value](int at_timestamp) {
MP_ASSERT_OK(graph.AddPacketToInputStream(
"tick",
MakePacket<int>(/*doesn't matter*/ 1).At(Timestamp(at_timestamp))));
MP_ASSERT_OK(graph.WaitUntilIdle());
ASSERT_FALSE(output_packets.empty());
EXPECT_EQ(Timestamp(at_timestamp), output_packets.back().Timestamp());
EXPECT_EQ(expected_value, output_packets.back().Get<int>());
};
tick_and_verify(/*at_timestamp=*/0);
tick_and_verify(/*at_timestamp=*/1);
tick_and_verify(/*at_timestamp=*/128);
tick_and_verify(/*at_timestamp=*/1024);
tick_and_verify(/*at_timestamp=*/1025);
}
TEST(SidePacketToStreamCalculator, AtTick_MultipleSidePackets) {
CalculatorGraphConfig graph_config =
ParseTextProtoOrDie<CalculatorGraphConfig>(
R"(
input_stream: "tick"
input_side_packet: "side_packet0"
input_side_packet: "side_packet1"
output_stream: "packet0"
output_stream: "packet1"
node {
calculator: "SidePacketToStreamCalculator"
input_stream: "TICK:tick"
input_side_packet: "side_packet0"
input_side_packet: "side_packet1"
output_stream: "AT_TICK:0:packet0"
output_stream: "AT_TICK:1:packet1"
}
)");
std::vector<Packet> output_packets0;
tool::AddVectorSink("packet0", &graph_config, &output_packets0);
std::vector<Packet> output_packets1;
tool::AddVectorSink("packet1", &graph_config, &output_packets1);
CalculatorGraph graph;
MP_ASSERT_OK(graph.Initialize(graph_config));
const int expected_value0 = 20;
const int expected_value1 = 128;
MP_ASSERT_OK(
graph.StartRun({{"side_packet0", MakePacket<int>(expected_value0)},
{"side_packet1", MakePacket<int>(expected_value1)}}));
auto tick_and_verify = [&graph, &output_packets0, &output_packets1,
expected_value0, expected_value1](int at_timestamp) {
MP_ASSERT_OK(graph.AddPacketToInputStream(
"tick",
MakePacket<int>(/*doesn't matter*/ 1).At(Timestamp(at_timestamp))));
MP_ASSERT_OK(graph.WaitUntilIdle());
ASSERT_FALSE(output_packets0.empty());
ASSERT_FALSE(output_packets1.empty());
EXPECT_EQ(Timestamp(at_timestamp), output_packets0.back().Timestamp());
EXPECT_EQ(expected_value0, output_packets0.back().Get<int>());
EXPECT_EQ(Timestamp(at_timestamp), output_packets1.back().Timestamp());
EXPECT_EQ(expected_value1, output_packets1.back().Get<int>());
};
tick_and_verify(/*at_timestamp=*/0);
tick_and_verify(/*at_timestamp=*/1);
tick_and_verify(/*at_timestamp=*/128);
tick_and_verify(/*at_timestamp=*/1024);
tick_and_verify(/*at_timestamp=*/1025);
}
} // namespace
} // namespace mediapipe
mediapipe/calculators/image/BUILD
+1 -3
View File
@@ -346,9 +346,7 @@ cc_library(
],
"//conditions:default": [],
}),
visibility = [
"//visibility:public",
],
visibility = ["//visibility:public"],
deps = [
":image_cropping_calculator_cc_proto",
"//mediapipe/framework:calculator_framework",
mediapipe/calculators/image/color_convert_calculator.cc
+5
View File
@@ -75,6 +75,11 @@ class ColorConvertCalculator : public CalculatorBase {
static ::mediapipe::Status GetContract(CalculatorContract* cc);
::mediapipe::Status Process(CalculatorContext* cc) override;
::mediapipe::Status Open(CalculatorContext* cc) override {
cc->SetOffset(TimestampDiff(0));
return ::mediapipe::OkStatus();
}
private:
// Wrangles the appropriate inputs and outputs to perform the color
// conversion. The ImageFrame on input_tag is converted using the
mediapipe/calculators/image/image_cropping_calculator.cc
+61 -1
View File
@@ -119,6 +119,13 @@ REGISTER_CALCULATOR(ImageCroppingCalculator);
#endif // !MEDIAPIPE_DISABLE_GPU
}
// Validate border mode.
if (use_gpu_) {
MP_RETURN_IF_ERROR(ValidateBorderModeForGPU(cc));
} else {
MP_RETURN_IF_ERROR(ValidateBorderModeForCPU(cc));
}
return ::mediapipe::OkStatus();
}
@@ -162,6 +169,32 @@ REGISTER_CALCULATOR(ImageCroppingCalculator);
return ::mediapipe::OkStatus();
}
::mediapipe::Status ImageCroppingCalculator::ValidateBorderModeForCPU(
CalculatorContext* cc) {
int border_mode;
return GetBorderModeForOpenCV(cc, &border_mode);
}
::mediapipe::Status ImageCroppingCalculator::ValidateBorderModeForGPU(
CalculatorContext* cc) {
mediapipe::ImageCroppingCalculatorOptions options =
cc->Options<mediapipe::ImageCroppingCalculatorOptions>();
switch (options.border_mode()) {
case mediapipe::ImageCroppingCalculatorOptions::BORDER_ZERO:
LOG(WARNING) << "BORDER_ZERO mode is not supported by GPU "
<< "implementation and will fall back into BORDER_REPLICATE";
break;
case mediapipe::ImageCroppingCalculatorOptions::BORDER_REPLICATE:
break;
default:
RET_CHECK_FAIL() << "Unsupported border mode for GPU: "
<< options.border_mode();
}
return ::mediapipe::OkStatus();
}
::mediapipe::Status ImageCroppingCalculator::RenderCpu(CalculatorContext* cc) {
if (cc->Inputs().Tag(kImageTag).IsEmpty()) {
return ::mediapipe::OkStatus();
@@ -172,6 +205,10 @@ REGISTER_CALCULATOR(ImageCroppingCalculator);
auto [target_width, target_height, rect_center_x, rect_center_y, rotation] =
GetCropSpecs(cc, input_img.Width(), input_img.Height());
// Get border mode and value for OpenCV.
int border_mode;
MP_RETURN_IF_ERROR(GetBorderModeForOpenCV(cc, &border_mode));
const cv::RotatedRect min_rect(cv::Point2f(rect_center_x, rect_center_y),
cv::Size2f(target_width, target_height),
rotation * 180.f / M_PI);
@@ -191,7 +228,9 @@ REGISTER_CALCULATOR(ImageCroppingCalculator);
cv::getPerspectiveTransform(src_points, dst_points);
cv::Mat cropped_image;
cv::warpPerspective(input_mat, cropped_image, projection_matrix,
cv::Size(min_rect.size.width, min_rect.size.height));
cv::Size(min_rect.size.width, min_rect.size.height),
/* flags = */ 0,
/* borderMode = */ border_mode);
std::unique_ptr<ImageFrame> output_frame(new ImageFrame(
input_img.Format(), cropped_image.cols, cropped_image.rows));
@@ -453,6 +492,7 @@ RectSpec ImageCroppingCalculator::GetCropSpecs(const CalculatorContext* cc,
rotation = options.rotation();
}
}
return {
.width = crop_width,
.height = crop_height,
@@ -462,4 +502,24 @@ RectSpec ImageCroppingCalculator::GetCropSpecs(const CalculatorContext* cc,
};
}
::mediapipe::Status ImageCroppingCalculator::GetBorderModeForOpenCV(
CalculatorContext* cc, int* border_mode) {
mediapipe::ImageCroppingCalculatorOptions options =
cc->Options<mediapipe::ImageCroppingCalculatorOptions>();
switch (options.border_mode()) {
case mediapipe::ImageCroppingCalculatorOptions::BORDER_ZERO:
*border_mode = cv::BORDER_CONSTANT;
break;
case mediapipe::ImageCroppingCalculatorOptions::BORDER_REPLICATE:
*border_mode = cv::BORDER_REPLICATE;
break;
default:
RET_CHECK_FAIL() << "Unsupported border mode for CPU: "
<< options.border_mode();
}
return ::mediapipe::OkStatus();
}
} // namespace mediapipe
mediapipe/calculators/image/image_cropping_calculator.h
+5
View File
@@ -36,6 +36,7 @@
// Note: input_stream values take precedence over options defined in the graph.
//
namespace mediapipe {
struct RectSpec {
int width;
int height;
@@ -63,12 +64,16 @@ class ImageCroppingCalculator : public CalculatorBase {
int src_height);
private:
::mediapipe::Status ValidateBorderModeForCPU(CalculatorContext* cc);
::mediapipe::Status ValidateBorderModeForGPU(CalculatorContext* cc);
::mediapipe::Status RenderCpu(CalculatorContext* cc);
::mediapipe::Status RenderGpu(CalculatorContext* cc);
::mediapipe::Status InitGpu(CalculatorContext* cc);
void GlRender();
void GetOutputDimensions(CalculatorContext* cc, int src_width, int src_height,
int* dst_width, int* dst_height);
::mediapipe::Status GetBorderModeForOpenCV(CalculatorContext* cc,
int* border_mode);
mediapipe::ImageCroppingCalculatorOptions options_;
mediapipe/calculators/image/image_cropping_calculator.proto
+11
View File
@@ -40,4 +40,15 @@ message ImageCroppingCalculatorOptions {
// The (0, 0) point is at the (top, left) corner.
optional float norm_center_x = 6 [default = 0];
optional float norm_center_y = 7 [default = 0];
enum BorderMode {
// First unspecified value is required by the guideline. See details here:
// https://developers.google.com/protocol-buffers/docs/style#enums
BORDER_UNSPECIFIED = 0;
BORDER_ZERO = 1;
BORDER_REPLICATE = 2;
}
// Specifies behaviour for crops that go beyond image borders.
optional BorderMode border_mode = 8 [default = BORDER_ZERO];
}
mediapipe/calculators/image/image_cropping_calculator_test.cc
+32 -32
View File
@@ -56,11 +56,11 @@ TEST(ImageCroppingCalculatorTest, GetCroppingDimensionsNormal) {
}
)");
auto calculator_state =
CalculatorState("Node", 0, "Calculator", calculator_node, nullptr);
auto cc =
CalculatorContext(&calculator_state, tool::CreateTagMap({}).ValueOrDie(),
tool::CreateTagMap({}).ValueOrDie());
auto calculator_state = absl::make_unique<CalculatorState>(
"Node", 0, "Calculator", calculator_node, nullptr);
auto cc = absl::make_unique<CalculatorContext>(
calculator_state.get(), tool::CreateTagMap({}).ValueOrDie(),
tool::CreateTagMap({}).ValueOrDie());
RectSpec expectRect = {
.width = 60,
@@ -69,9 +69,9 @@ TEST(ImageCroppingCalculatorTest, GetCroppingDimensionsNormal) {
.center_y = 50,
.rotation = 0.3,
};
EXPECT_EQ(
ImageCroppingCalculator::GetCropSpecs(&cc, input_width, input_height),
expectRect);
EXPECT_EQ(ImageCroppingCalculator::GetCropSpecs(cc.get(), input_width,
input_height),
expectRect);
} // TEST
// Test when (width height) + (norm_width norm_height) are set in options.
@@ -96,11 +96,11 @@ TEST(ImageCroppingCalculatorTest, RedundantSpecInOptions) {
}
)");
auto calculator_state =
CalculatorState("Node", 0, "Calculator", calculator_node, nullptr);
auto cc =
CalculatorContext(&calculator_state, tool::CreateTagMap({}).ValueOrDie(),
tool::CreateTagMap({}).ValueOrDie());
auto calculator_state = absl::make_unique<CalculatorState>(
"Node", 0, "Calculator", calculator_node, nullptr);
auto cc = absl::make_unique<CalculatorContext>(
calculator_state.get(), tool::CreateTagMap({}).ValueOrDie(),
tool::CreateTagMap({}).ValueOrDie());
RectSpec expectRect = {
.width = 50,
.height = 50,
@@ -108,9 +108,9 @@ TEST(ImageCroppingCalculatorTest, RedundantSpecInOptions) {
.center_y = 50,
.rotation = 0.3,
};
EXPECT_EQ(
ImageCroppingCalculator::GetCropSpecs(&cc, input_width, input_height),
expectRect);
EXPECT_EQ(ImageCroppingCalculator::GetCropSpecs(cc.get(), input_width,
input_height),
expectRect);
} // TEST
// Test when WIDTH HEIGHT are set from input stream,
@@ -138,16 +138,16 @@ TEST(ImageCroppingCalculatorTest, RedundantSpectWithInputStream) {
}
)");
auto calculator_state =
CalculatorState("Node", 0, "Calculator", calculator_node, nullptr);
auto calculator_state = absl::make_unique<CalculatorState>(
"Node", 0, "Calculator", calculator_node, nullptr);
auto inputTags = tool::CreateTagMap({
"HEIGHT:0:crop_height",
"WIDTH:0:crop_width",
})
.ValueOrDie();
auto cc = CalculatorContext(&calculator_state, inputTags,
tool::CreateTagMap({}).ValueOrDie());
auto& inputs = cc.Inputs();
auto cc = absl::make_unique<CalculatorContext>(
calculator_state.get(), inputTags, tool::CreateTagMap({}).ValueOrDie());
auto& inputs = cc->Inputs();
inputs.Tag(kHeightTag).Value() = MakePacket<int>(1);
inputs.Tag(kWidthTag).Value() = MakePacket<int>(1);
RectSpec expectRect = {
@@ -157,9 +157,9 @@ TEST(ImageCroppingCalculatorTest, RedundantSpectWithInputStream) {
.center_y = 50,
.rotation = 0.3,
};
EXPECT_EQ(
ImageCroppingCalculator::GetCropSpecs(&cc, input_width, input_height),
expectRect);
EXPECT_EQ(ImageCroppingCalculator::GetCropSpecs(cc.get(), input_width,
input_height),
expectRect);
} // TEST
// Test when RECT is set from input stream,
@@ -186,15 +186,15 @@ TEST(ImageCroppingCalculatorTest, RedundantSpecWithInputStream) {
}
)");
auto calculator_state =
CalculatorState("Node", 0, "Calculator", calculator_node, nullptr);
auto calculator_state = absl::make_unique<CalculatorState>(
"Node", 0, "Calculator", calculator_node, nullptr);
auto inputTags = tool::CreateTagMap({
"RECT:0:rect",
})
.ValueOrDie();
auto cc = CalculatorContext(&calculator_state, inputTags,
tool::CreateTagMap({}).ValueOrDie());
auto& inputs = cc.Inputs();
auto cc = absl::make_unique<CalculatorContext>(
calculator_state.get(), inputTags, tool::CreateTagMap({}).ValueOrDie());
auto& inputs = cc->Inputs();
mediapipe::Rect rect = ParseTextProtoOrDie<mediapipe::Rect>(
R"(
width: 1 height: 1 x_center: 40 y_center: 40 rotation: 0.5
@@ -207,9 +207,9 @@ TEST(ImageCroppingCalculatorTest, RedundantSpecWithInputStream) {
.center_y = 40,
.rotation = 0.5,
};
EXPECT_EQ(
ImageCroppingCalculator::GetCropSpecs(&cc, input_width, input_height),
expectRect);
EXPECT_EQ(ImageCroppingCalculator::GetCropSpecs(cc.get(), input_width,
input_height),
expectRect);
} // TEST
} // namespace
mediapipe/calculators/image/image_transformation_calculator.cc
+71 -15
View File
@@ -104,6 +104,14 @@ mediapipe::ScaleMode_Mode ParseScaleMode(
// to be a multiple of 90 degrees. If provided, it overrides the
// ROTATION_DEGREES input side packet.
//
// FLIP_HORIZONTALLY (optional): Whether to flip image horizontally or not. If
// provided, it overrides the FLIP_HORIZONTALLY input side packet and/or
// corresponding field in the calculator options.
//
// FLIP_VERTICALLY (optional): Whether to flip image vertically or not. If
// provided, it overrides the FLIP_VERTICALLY input side packet and/or
// corresponding field in the calculator options.
//
// Output:
// One of the following two tags:
// IMAGE - ImageFrame representing the output image.
@@ -129,6 +137,12 @@ mediapipe::ScaleMode_Mode ParseScaleMode(
// degrees. It has to be a multiple of 90 degrees. It overrides the
// corresponding field in the calculator options.
//
// FLIP_HORIZONTALLY (optional): Whether to flip image horizontally or not.
// It overrides the corresponding field in the calculator options.
//
// FLIP_VERTICALLY (optional): Whether to flip image vertically or not.
// It overrides the corresponding field in the calculator options.
//
// Calculator options (see image_transformation_calculator.proto):
// output_width, output_height - (optional) Desired scaled image size.
// rotation_mode - (optional) Rotation in multiples of 90 degrees.
@@ -167,6 +181,8 @@ class ImageTransformationCalculator : public CalculatorBase {
int output_height_ = 0;
mediapipe::RotationMode_Mode rotation_;
mediapipe::ScaleMode_Mode scale_mode_;
bool flip_horizontally_ = false;
bool flip_vertically_ = false;
bool use_gpu_ = false;
#if !defined(MEDIAPIPE_DISABLE_GPU)
@@ -203,6 +219,12 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
if (cc->Inputs().HasTag("ROTATION_DEGREES")) {
cc->Inputs().Tag("ROTATION_DEGREES").Set<int>();
}
if (cc->Inputs().HasTag("FLIP_HORIZONTALLY")) {
cc->Inputs().Tag("FLIP_HORIZONTALLY").Set<bool>();
}
if (cc->Inputs().HasTag("FLIP_VERTICALLY")) {
cc->Inputs().Tag("FLIP_VERTICALLY").Set<bool>();
}
if (cc->InputSidePackets().HasTag("OUTPUT_DIMENSIONS")) {
cc->InputSidePackets().Tag("OUTPUT_DIMENSIONS").Set<DimensionsPacketType>();
@@ -210,6 +232,12 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
if (cc->InputSidePackets().HasTag("ROTATION_DEGREES")) {
cc->InputSidePackets().Tag("ROTATION_DEGREES").Set<int>();
}
if (cc->InputSidePackets().HasTag("FLIP_HORIZONTALLY")) {
cc->InputSidePackets().Tag("FLIP_HORIZONTALLY").Set<bool>();
}
if (cc->InputSidePackets().HasTag("FLIP_VERTICALLY")) {
cc->InputSidePackets().Tag("FLIP_VERTICALLY").Set<bool>();
}
if (cc->Outputs().HasTag("LETTERBOX_PADDING")) {
cc->Outputs().Tag("LETTERBOX_PADDING").Set<std::array<float, 4>>();
@@ -245,6 +273,7 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
output_width_ = options_.output_width();
output_height_ = options_.output_height();
}
if (cc->InputSidePackets().HasTag("ROTATION_DEGREES")) {
rotation_ = DegreesToRotationMode(
cc->InputSidePackets().Tag("ROTATION_DEGREES").Get<int>());
@@ -252,6 +281,20 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
rotation_ = options_.rotation_mode();
}
if (cc->InputSidePackets().HasTag("FLIP_HORIZONTALLY")) {
flip_horizontally_ =
cc->InputSidePackets().Tag("FLIP_HORIZONTALLY").Get<bool>();
} else {
flip_horizontally_ = options_.flip_horizontally();
}
if (cc->InputSidePackets().HasTag("FLIP_VERTICALLY")) {
flip_vertically_ =
cc->InputSidePackets().Tag("FLIP_VERTICALLY").Get<bool>();
} else {
flip_vertically_ = options_.flip_vertically();
}
scale_mode_ = ParseScaleMode(options_.scale_mode(), DEFAULT_SCALE_MODE);
if (use_gpu_) {
@@ -268,12 +311,37 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
::mediapipe::Status ImageTransformationCalculator::Process(
CalculatorContext* cc) {
// Override values if specified so.
if (cc->Inputs().HasTag("ROTATION_DEGREES") &&
!cc->Inputs().Tag("ROTATION_DEGREES").IsEmpty()) {
rotation_ =
DegreesToRotationMode(cc->Inputs().Tag("ROTATION_DEGREES").Get<int>());
}
if (cc->Inputs().HasTag("FLIP_HORIZONTALLY") &&
!cc->Inputs().Tag("FLIP_HORIZONTALLY").IsEmpty()) {
flip_horizontally_ = cc->Inputs().Tag("FLIP_HORIZONTALLY").Get<bool>();
}
if (cc->Inputs().HasTag("FLIP_VERTICALLY") &&
!cc->Inputs().Tag("FLIP_VERTICALLY").IsEmpty()) {
flip_vertically_ = cc->Inputs().Tag("FLIP_VERTICALLY").Get<bool>();
}
if (use_gpu_) {
#if !defined(MEDIAPIPE_DISABLE_GPU)
if (cc->Inputs().Tag("IMAGE_GPU").IsEmpty()) {
// Image is missing, hence no way to produce output image. (Timestamp
// bound will be updated automatically.)
return ::mediapipe::OkStatus();
}
return helper_.RunInGlContext(
[this, cc]() -> ::mediapipe::Status { return RenderGpu(cc); });
#endif // !MEDIAPIPE_DISABLE_GPU
} else {
if (cc->Inputs().Tag("IMAGE").IsEmpty()) {
// Image is missing, hence no way to produce output image. (Timestamp
// bound will be updated automatically.)
return ::mediapipe::OkStatus();
}
return RenderCpu(cc);
}
return ::mediapipe::OkStatus();
@@ -360,11 +428,6 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
.Add(padding.release(), cc->InputTimestamp());
}
if (cc->InputSidePackets().HasTag("ROTATION_DEGREES")) {
rotation_ = DegreesToRotationMode(
cc->InputSidePackets().Tag("ROTATION_DEGREES").Get<int>());
}
cv::Mat rotated_mat;
const int angle = RotationModeToDegrees(rotation_);
cv::Point2f src_center(scaled_mat.cols / 2.0, scaled_mat.rows / 2.0);
@@ -372,11 +435,9 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
cv::warpAffine(scaled_mat, rotated_mat, rotation_mat, scaled_mat.size());
cv::Mat flipped_mat;
if (options_.flip_horizontally() || options_.flip_vertically()) {
if (flip_horizontally_ || flip_vertically_) {
const int flip_code =
options_.flip_horizontally() && options_.flip_vertically()
? -1
: options_.flip_horizontally();
flip_horizontally_ && flip_vertically_ ? -1 : flip_horizontally_;
cv::flip(rotated_mat, flipped_mat, flip_code);
} else {
flipped_mat = rotated_mat;
@@ -450,11 +511,6 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
}
RET_CHECK(renderer) << "Unsupported input texture type";
if (cc->InputSidePackets().HasTag("ROTATION_DEGREES")) {
rotation_ = DegreesToRotationMode(
cc->InputSidePackets().Tag("ROTATION_DEGREES").Get<int>());
}
mediapipe::FrameScaleMode scale_mode = mediapipe::FrameScaleModeFromProto(
scale_mode_, mediapipe::FrameScaleMode::kStretch);
mediapipe::FrameRotation rotation =
@@ -469,7 +525,7 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
MP_RETURN_IF_ERROR(renderer->GlRender(
src1.width(), src1.height(), dst.width(), dst.height(), scale_mode,
rotation, options_.flip_horizontally(), options_.flip_vertically(),
rotation, flip_horizontally_, flip_vertically_,
/*flip_texture=*/false));
glActiveTexture(GL_TEXTURE1);
mediapipe/calculators/image/scale_image_calculator.cc
+14 -10
View File
@@ -260,11 +260,11 @@ ScaleImageCalculator::~ScaleImageCalculator() {}
&crop_width_, &crop_height_, //
&col_start_, &row_start_));
MP_RETURN_IF_ERROR(
scale_image::FindOutputDimensions(crop_width_, crop_height_, //
options_.target_width(), //
options_.target_height(), //
options_.preserve_aspect_ratio(), //
options_.scale_to_multiple_of_two(), //
scale_image::FindOutputDimensions(crop_width_, crop_height_, //
options_.target_width(), //
options_.target_height(), //
options_.preserve_aspect_ratio(), //
options_.scale_to_multiple_of(), //
&output_width_, &output_height_));
MP_RETURN_IF_ERROR(FindInterpolationAlgorithm(options_.algorithm(),
&interpolation_algorithm_));
@@ -361,17 +361,21 @@ ScaleImageCalculator::~ScaleImageCalculator() {}
output_format_ = input_format_;
}
const bool is_positive_and_even =
(options_.scale_to_multiple_of() >= 1) &&
(options_.scale_to_multiple_of() % 2 == 0);
if (output_format_ == ImageFormat::YCBCR420P) {
RET_CHECK(options_.scale_to_multiple_of_two())
RET_CHECK(is_positive_and_even)
<< "ScaleImageCalculator always outputs width and height that are "
"divisible by 2 when output format is YCbCr420P. To scale to "
"width and height of odd numbers, the output format must be SRGB.";
} else if (options_.preserve_aspect_ratio()) {
RET_CHECK(options_.scale_to_multiple_of_two())
RET_CHECK(options_.scale_to_multiple_of() == 2)
<< "ScaleImageCalculator always outputs width and height that are "
"divisible by 2 when perserving aspect ratio. To scale to width "
"and height of odd numbers, please set "
"preserve_aspect_ratio to false.";
"divisible by 2 when preserving aspect ratio. If you'd like to "
"set scale_to_multiple_of to something other than 2, please "
"set preserve_aspect_ratio to false.";
}
if (input_width_ > 0 && input_height_ > 0 &&
mediapipe/calculators/image/scale_image_calculator.proto
+14 -10
View File
@@ -11,9 +11,10 @@ import "mediapipe/framework/formats/image_format.proto";
// 2) Scale and convert the image to fit inside target_width x target_height
// using the specified scaling algorithm. (maintaining the aspect
// ratio if preserve_aspect_ratio is true).
// The output width and height will be divisible by 2. It is possible to output
// width and height that are odd number when the output format is SRGB and not
// perserving the aspect ratio. See scale_to_multiple_of_two option for details.
// The output width and height will be divisible by 2, by default. It is
// possible to output width and height that are odd numbers when the output
// format is SRGB and the aspect ratio is left unpreserved. See
// scale_to_multiple_of for details.
message ScaleImageCalculatorOptions {
extend CalculatorOptions {
optional ScaleImageCalculatorOptions ext = 66237115;
@@ -23,7 +24,7 @@ message ScaleImageCalculatorOptions {
// depending on the other options below. If unset, use the same width
// or height as the input. If only one is set then determine the other
// from the aspect ratio (after cropping). The output width and height
// will be divisible by 2.
// will be divisible by 2, by default.
optional int32 target_width = 1;
optional int32 target_height = 2;
@@ -31,7 +32,8 @@ message ScaleImageCalculatorOptions {
// fits inside the box represented by target_width and target_height.
// Otherwise it is scaled to fit target_width and target_height
// completely. In any case, the aspect ratio that is preserved is
// that after cropping to the minimum/maximum aspect ratio.
// that after cropping to the minimum/maximum aspect ratio. Additionally, if
// true, the output width and height will be divisible by 2.
optional bool preserve_aspect_ratio = 3 [default = true];
// If ratio is positive, crop the image to this minimum and maximum
@@ -95,11 +97,13 @@ message ScaleImageCalculatorOptions {
// SRGB or YCBCR420P.
optional ImageFormat.Format input_format = 12;
// If true, the output width and height will be divisible by 2. Otherwise it
// will use the exact specified output width and height, which is only
// supported when the output format is SRGB and preserve_aspect_ratio option
// is set to false.
optional bool scale_to_multiple_of_two = 13 [default = true];
// If set to 2, the target width and height will be rounded-down
// to the nearest even number. If set to any positive value other than 2,
// preserve_aspect_ratio must be false and the target width and height will be
// rounded-down to multiples of the given value. If set to any value less than
// 1, it will be treated like 1.
// NOTE: If set to an odd number, the output format must be SRGB.
optional int32 scale_to_multiple_of = 13 [default = 2];
// If true, assume the input YUV is BT.709 (this is the HDTV standard, so most
// content is likely using it). If false use the previous assumption of BT.601
mediapipe/calculators/image/scale_image_utils.cc
+23 -13
View File
@@ -88,17 +88,27 @@ double ParseRational(const std::string& rational) {
return ::mediapipe::OkStatus();
}
::mediapipe::Status FindOutputDimensions(int input_width, //
int input_height, //
int target_width, //
int target_height, //
bool preserve_aspect_ratio, //
bool scale_to_multiple_of_two, //
::mediapipe::Status FindOutputDimensions(int input_width, //
int input_height, //
int target_width, //
int target_height, //
bool preserve_aspect_ratio, //
int scale_to_multiple_of, //
int* output_width,
int* output_height) {
CHECK(output_width);
CHECK(output_height);
if (preserve_aspect_ratio) {
RET_CHECK(scale_to_multiple_of == 2)
<< "FindOutputDimensions always outputs width and height that are "
"divisible by 2 when preserving aspect ratio. If you'd like to "
"set scale_to_multiple_of to something other than 2, please "
"set preserve_aspect_ratio to false.";
}
if (scale_to_multiple_of < 1) scale_to_multiple_of = 1;
if (!preserve_aspect_ratio || (target_width <= 0 && target_height <= 0)) {
if (target_width <= 0) {
target_width = input_width;
@@ -106,13 +116,13 @@ double ParseRational(const std::string& rational) {
if (target_height <= 0) {
target_height = input_height;
}
if (scale_to_multiple_of_two) {
*output_width = (target_width / 2) * 2;
*output_height = (target_height / 2) * 2;
} else {
*output_width = target_width;
*output_height = target_height;
}
target_width -= target_width % scale_to_multiple_of;
target_height -= target_height % scale_to_multiple_of;
*output_width = target_width;
*output_height = target_height;
return ::mediapipe::OkStatus();
}
mediapipe/calculators/image/scale_image_utils.h
+11 -9
View File
@@ -35,17 +35,19 @@ namespace scale_image {
int* col_start, int* row_start);
// Given an input width and height, a target width and height, whether to
// preserve the aspect ratio, and whether to round down to a multiple of 2,
// determine the output width and height. If target_width or target_height is
// non-positive, then they will be set to the input_width and input_height
// respectively. The output_width and output_height will be reduced as necessary
// to preserve_aspect_ratio and to scale_to_multipe_of_two if these options are
// specified.
// preserve the aspect ratio, and whether to round-down to the multiple of a
// given number nearest to the targets, determine the output width and height.
// If target_width or target_height is non-positive, then they will be set to
// the input_width and input_height respectively. If scale_to_multiple_of is
// less than 1, it will be treated like 1. The output_width and
// output_height will be reduced as necessary to preserve_aspect_ratio if the
// option is specified. If preserving the aspect ratio is desired, you must set
// scale_to_multiple_of to 2.
::mediapipe::Status FindOutputDimensions(int input_width, int input_height, //
int target_width,
int target_height, //
bool preserve_aspect_ratio, //
bool scale_to_multiple_of_two, //
int target_height, //
bool preserve_aspect_ratio, //
int scale_to_multiple_of, //
int* output_width, int* output_height);
} // namespace scale_image
mediapipe/calculators/image/scale_image_utils_test.cc
+63 -23
View File
@@ -79,49 +79,49 @@ TEST(ScaleImageUtilsTest, FindOutputDimensionsPreserveRatio) {
int output_width;
int output_height;
// Not scale.
MP_ASSERT_OK(FindOutputDimensions(200, 100, -1, -1, true, true, &output_width,
MP_ASSERT_OK(FindOutputDimensions(200, 100, -1, -1, true, 2, &output_width,
&output_height));
EXPECT_EQ(200, output_width);
EXPECT_EQ(100, output_height);
// Not scale with odd input size.
MP_ASSERT_OK(FindOutputDimensions(201, 101, -1, -1, false, false,
&output_width, &output_height));
MP_ASSERT_OK(FindOutputDimensions(201, 101, -1, -1, false, 1, &output_width,
&output_height));
EXPECT_EQ(201, output_width);
EXPECT_EQ(101, output_height);
// Scale down by 1/2.
MP_ASSERT_OK(FindOutputDimensions(200, 100, 100, -1, true, true,
&output_width, &output_height));
MP_ASSERT_OK(FindOutputDimensions(200, 100, 100, -1, true, 2, &output_width,
&output_height));
EXPECT_EQ(100, output_width);
EXPECT_EQ(50, output_height);
// Scale up, doubling dimensions.
MP_ASSERT_OK(FindOutputDimensions(200, 100, -1, 200, true, true,
&output_width, &output_height));
MP_ASSERT_OK(FindOutputDimensions(200, 100, -1, 200, true, 2, &output_width,
&output_height));
EXPECT_EQ(400, output_width);
EXPECT_EQ(200, output_height);
// Fits a 2:1 image into a 150 x 150 box. Output dimensions are always
// visible by 2.
MP_ASSERT_OK(FindOutputDimensions(200, 100, 150, 150, true, true,
&output_width, &output_height));
MP_ASSERT_OK(FindOutputDimensions(200, 100, 150, 150, true, 2, &output_width,
&output_height));
EXPECT_EQ(150, output_width);
EXPECT_EQ(74, output_height);
// Fits a 2:1 image into a 400 x 50 box.
MP_ASSERT_OK(FindOutputDimensions(200, 100, 400, 50, true, true,
&output_width, &output_height));
MP_ASSERT_OK(FindOutputDimensions(200, 100, 400, 50, true, 2, &output_width,
&output_height));
EXPECT_EQ(100, output_width);
EXPECT_EQ(50, output_height);
// Scale to multiple number with odd targe size.
MP_ASSERT_OK(FindOutputDimensions(200, 100, 101, -1, true, true,
&output_width, &output_height));
MP_ASSERT_OK(FindOutputDimensions(200, 100, 101, -1, true, 2, &output_width,
&output_height));
EXPECT_EQ(100, output_width);
EXPECT_EQ(50, output_height);
// Scale to multiple number with odd targe size.
MP_ASSERT_OK(FindOutputDimensions(200, 100, 101, -1, true, false,
&output_width, &output_height));
MP_ASSERT_OK(FindOutputDimensions(200, 100, 101, -1, true, 2, &output_width,
&output_height));
EXPECT_EQ(100, output_width);
EXPECT_EQ(50, output_height);
// Scale to odd size.
MP_ASSERT_OK(FindOutputDimensions(200, 100, 151, 101, false, false,
&output_width, &output_height));
MP_ASSERT_OK(FindOutputDimensions(200, 100, 151, 101, false, 1, &output_width,
&output_height));
EXPECT_EQ(151, output_width);
EXPECT_EQ(101, output_height);
}
@@ -131,22 +131,62 @@ TEST(ScaleImageUtilsTest, FindOutputDimensionsNoAspectRatio) {
int output_width;
int output_height;
// Scale width only.
MP_ASSERT_OK(FindOutputDimensions(200, 100, 100, -1, false, true,
&output_width, &output_height));
MP_ASSERT_OK(FindOutputDimensions(200, 100, 100, -1, false, 2, &output_width,
&output_height));
EXPECT_EQ(100, output_width);
EXPECT_EQ(100, output_height);
// Scale height only.
MP_ASSERT_OK(FindOutputDimensions(200, 100, -1, 200, false, true,
&output_width, &output_height));
MP_ASSERT_OK(FindOutputDimensions(200, 100, -1, 200, false, 2, &output_width,
&output_height));
EXPECT_EQ(200, output_width);
EXPECT_EQ(200, output_height);
// Scale both dimensions.
MP_ASSERT_OK(FindOutputDimensions(200, 100, 150, 200, false, true,
&output_width, &output_height));
MP_ASSERT_OK(FindOutputDimensions(200, 100, 150, 200, false, 2, &output_width,
&output_height));
EXPECT_EQ(150, output_width);
EXPECT_EQ(200, output_height);
}
// Tests scale_to_multiple_of.
TEST(ScaleImageUtilsTest, FindOutputDimensionsDownScaleToMultipleOf) {
int output_width;
int output_height;
// Set no targets, downscale to a multiple of 8.
MP_ASSERT_OK(FindOutputDimensions(100, 100, -1, -1, false, 8, &output_width,
&output_height));
EXPECT_EQ(96, output_width);
EXPECT_EQ(96, output_height);
// Set width target, downscale to a multiple of 8.
MP_ASSERT_OK(FindOutputDimensions(200, 100, 100, -1, false, 8, &output_width,
&output_height));
EXPECT_EQ(96, output_width);
EXPECT_EQ(96, output_height);
// Set height target, downscale to a multiple of 8.
MP_ASSERT_OK(FindOutputDimensions(201, 101, -1, 201, false, 8, &output_width,
&output_height));
EXPECT_EQ(200, output_width);
EXPECT_EQ(200, output_height);
// Set both targets, downscale to a multiple of 8.
MP_ASSERT_OK(FindOutputDimensions(200, 100, 150, 200, false, 8, &output_width,
&output_height));
EXPECT_EQ(144, output_width);
EXPECT_EQ(200, output_height);
// Doesn't throw error if keep aspect is true and downscale multiple is 2.
MP_ASSERT_OK(FindOutputDimensions(200, 100, 400, 200, true, 2, &output_width,
&output_height));
EXPECT_EQ(400, output_width);
EXPECT_EQ(200, output_height);
// Throws error if keep aspect is true, but downscale multiple is not 2.
ASSERT_THAT(FindOutputDimensions(200, 100, 400, 200, true, 4, &output_width,
&output_height),
testing::Not(testing::status::IsOk()));
// Downscaling to multiple ignored if multiple is less than 2.
MP_ASSERT_OK(FindOutputDimensions(200, 100, 401, 201, false, 1, &output_width,
&output_height));
EXPECT_EQ(401, output_width);
EXPECT_EQ(201, output_height);
}
} // namespace
} // namespace scale_image
} // namespace mediapipe
mediapipe/calculators/tensorflow/BUILD
+31 -21
View File
@@ -138,7 +138,7 @@ mediapipe_cc_proto_library(
srcs = ["image_frame_to_tensor_calculator.proto"],
cc_deps = [
"//mediapipe/framework:calculator_cc_proto",
"@org_tensorflow//tensorflow/core:protos_all",
"@org_tensorflow//tensorflow/core:protos_all_cc",
],
visibility = ["//visibility:public"],
deps = [":image_frame_to_tensor_calculator_proto"],
@@ -173,7 +173,7 @@ mediapipe_cc_proto_library(
srcs = ["pack_media_sequence_calculator.proto"],
cc_deps = [
"//mediapipe/framework:calculator_cc_proto",
"@org_tensorflow//tensorflow/core:protos_all",
"@org_tensorflow//tensorflow/core:protos_all_cc",
],
visibility = ["//visibility:public"],
deps = [":pack_media_sequence_calculator_proto"],
@@ -192,7 +192,7 @@ mediapipe_cc_proto_library(
srcs = ["tensorflow_session_from_frozen_graph_generator.proto"],
cc_deps = [
"//mediapipe/framework:packet_generator_cc_proto",
"@org_tensorflow//tensorflow/core:protos_all",
"@org_tensorflow//tensorflow/core:protos_all_cc",
],
visibility = ["//visibility:public"],
deps = [":tensorflow_session_from_frozen_graph_generator_proto"],
@@ -203,7 +203,7 @@ mediapipe_cc_proto_library(
srcs = ["tensorflow_session_from_frozen_graph_calculator.proto"],
cc_deps = [
"//mediapipe/framework:calculator_cc_proto",
"@org_tensorflow//tensorflow/core:protos_all",
"@org_tensorflow//tensorflow/core:protos_all_cc",
],
visibility = ["//visibility:public"],
deps = [":tensorflow_session_from_frozen_graph_calculator_proto"],
@@ -277,7 +277,7 @@ mediapipe_cc_proto_library(
srcs = ["vector_int_to_tensor_calculator_options.proto"],
cc_deps = [
"//mediapipe/framework:calculator_cc_proto",
"@org_tensorflow//tensorflow/core:protos_all",
"@org_tensorflow//tensorflow/core:protos_all_cc",
],
visibility = ["//visibility:public"],
deps = [":vector_int_to_tensor_calculator_options_proto"],
@@ -408,7 +408,7 @@ cc_library(
"//mediapipe/util/sequence:media_sequence",
"//mediapipe/util/sequence:media_sequence_util",
"@com_google_absl//absl/strings",
"@org_tensorflow//tensorflow/core:protos_all",
"@org_tensorflow//tensorflow/core:protos_all_cc",
],
alwayslink = 1,
)
@@ -423,7 +423,7 @@ cc_library(
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"@org_tensorflow//tensorflow/core:protos_all",
"@org_tensorflow//tensorflow/core:protos_all_cc",
],
alwayslink = 1,
)
@@ -654,7 +654,7 @@ cc_library(
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"@org_tensorflow//tensorflow/core:lib",
"@org_tensorflow//tensorflow/core:protos_all",
"@org_tensorflow//tensorflow/core:protos_all_cc",
],
alwayslink = 1,
)
@@ -695,7 +695,7 @@ cc_library(
"//mediapipe/util:audio_decoder_cc_proto",
"//mediapipe/util/sequence:media_sequence",
"@com_google_absl//absl/strings",
"@org_tensorflow//tensorflow/core:protos_all",
"@org_tensorflow//tensorflow/core:protos_all_cc",
],
alwayslink = 1,
)
@@ -737,7 +737,7 @@ cc_library(
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:packet",
"//mediapipe/framework/port:status",
"@org_tensorflow//tensorflow/core:protos_all",
"@org_tensorflow//tensorflow/core:protos_all_cc",
],
alwayslink = 1,
)
@@ -745,6 +745,7 @@ cc_library(
cc_test(
name = "graph_tensors_packet_generator_test",
srcs = ["graph_tensors_packet_generator_test.cc"],
linkstatic = 1,
deps = [
":graph_tensors_packet_generator",
":graph_tensors_packet_generator_cc_proto",
@@ -761,6 +762,7 @@ cc_test(
name = "image_frame_to_tensor_calculator_test",
size = "small",
srcs = ["image_frame_to_tensor_calculator_test.cc"],
linkstatic = 1,
deps = [
":image_frame_to_tensor_calculator",
"//mediapipe/framework:calculator_framework",
@@ -777,6 +779,7 @@ cc_test(
name = "matrix_to_tensor_calculator_test",
size = "small",
srcs = ["matrix_to_tensor_calculator_test.cc"],
linkstatic = 1,
deps = [
":matrix_to_tensor_calculator",
":matrix_to_tensor_calculator_options_cc_proto",
@@ -793,6 +796,7 @@ cc_test(
name = "lapped_tensor_buffer_calculator_test",
size = "small",
srcs = ["lapped_tensor_buffer_calculator_test.cc"],
linkstatic = 1,
deps = [
":lapped_tensor_buffer_calculator",
":lapped_tensor_buffer_calculator_cc_proto",
@@ -801,7 +805,7 @@ cc_test(
"//mediapipe/framework/port:gtest_main",
"@com_google_absl//absl/memory",
"@org_tensorflow//tensorflow/core:framework",
"@org_tensorflow//tensorflow/core:protos_all",
"@org_tensorflow//tensorflow/core:protos_all_cc",
],
)
@@ -840,7 +844,7 @@ cc_test(
"//mediapipe/util/sequence:media_sequence",
"@com_google_absl//absl/memory",
"@com_google_absl//absl/strings",
"@org_tensorflow//tensorflow/core:protos_all",
"@org_tensorflow//tensorflow/core:protos_all_cc",
],
)
@@ -867,7 +871,7 @@ cc_test(
"@com_google_absl//absl/strings",
"@org_tensorflow//tensorflow/core:direct_session",
"@org_tensorflow//tensorflow/core:framework",
"@org_tensorflow//tensorflow/core:protos_all",
"@org_tensorflow//tensorflow/core:protos_all_cc",
"@org_tensorflow//tensorflow/core:testlib",
"@org_tensorflow//tensorflow/core/kernels:conv_ops",
"@org_tensorflow//tensorflow/core/kernels:math",
@@ -897,7 +901,7 @@ cc_test(
"@com_google_absl//absl/strings",
"@org_tensorflow//tensorflow/core:direct_session",
"@org_tensorflow//tensorflow/core:framework",
"@org_tensorflow//tensorflow/core:protos_all",
"@org_tensorflow//tensorflow/core:protos_all_cc",
"@org_tensorflow//tensorflow/core:testlib",
"@org_tensorflow//tensorflow/core/kernels:conv_ops",
"@org_tensorflow//tensorflow/core/kernels:math",
@@ -956,6 +960,7 @@ cc_test(
cc_test(
name = "tensor_squeeze_dimensions_calculator_test",
srcs = ["tensor_squeeze_dimensions_calculator_test.cc"],
linkstatic = 1,
deps = [
":tensor_squeeze_dimensions_calculator",
":tensor_squeeze_dimensions_calculator_cc_proto",
@@ -963,7 +968,7 @@ cc_test(
"//mediapipe/framework:calculator_runner",
"//mediapipe/framework/port:gtest_main",
"@org_tensorflow//tensorflow/core:framework",
"@org_tensorflow//tensorflow/core:protos_all",
"@org_tensorflow//tensorflow/core:protos_all_cc",
],
)
@@ -971,6 +976,7 @@ cc_test(
name = "tensor_to_image_frame_calculator_test",
size = "small",
srcs = ["tensor_to_image_frame_calculator_test.cc"],
linkstatic = 1,
deps = [
":tensor_to_image_frame_calculator",
":tensor_to_image_frame_calculator_cc_proto",
@@ -979,7 +985,7 @@ cc_test(
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/port:gtest_main",
"@org_tensorflow//tensorflow/core:framework",
"@org_tensorflow//tensorflow/core:protos_all",
"@org_tensorflow//tensorflow/core:protos_all_cc",
],
)
@@ -987,6 +993,7 @@ cc_test(
name = "tensor_to_matrix_calculator_test",
size = "small",
srcs = ["tensor_to_matrix_calculator_test.cc"],
linkstatic = 1,
deps = [
":tensor_to_matrix_calculator",
":tensor_to_matrix_calculator_cc_proto",
@@ -996,13 +1003,14 @@ cc_test(
"//mediapipe/framework/formats:time_series_header_cc_proto",
"//mediapipe/framework/port:gtest_main",
"@org_tensorflow//tensorflow/core:framework",
"@org_tensorflow//tensorflow/core:protos_all",
"@org_tensorflow//tensorflow/core:protos_all_cc",
],
)
cc_test(
name = "tensor_to_vector_float_calculator_test",
srcs = ["tensor_to_vector_float_calculator_test.cc"],
linkstatic = 1,
deps = [
":tensor_to_vector_float_calculator",
":tensor_to_vector_float_calculator_options_cc_proto",
@@ -1010,7 +1018,7 @@ cc_test(
"//mediapipe/framework:calculator_runner",
"//mediapipe/framework/port:gtest_main",
"@org_tensorflow//tensorflow/core:framework",
"@org_tensorflow//tensorflow/core:protos_all",
"@org_tensorflow//tensorflow/core:protos_all_cc",
],
)
@@ -1030,13 +1038,14 @@ cc_test(
"//mediapipe/util/sequence:media_sequence",
"@com_google_absl//absl/memory",
"@com_google_absl//absl/strings",
"@org_tensorflow//tensorflow/core:protos_all",
"@org_tensorflow//tensorflow/core:protos_all_cc",
],
)
cc_test(
name = "vector_int_to_tensor_calculator_test",
srcs = ["vector_int_to_tensor_calculator_test.cc"],
linkstatic = 1,
deps = [
":vector_int_to_tensor_calculator",
":vector_int_to_tensor_calculator_options_cc_proto",
@@ -1044,13 +1053,14 @@ cc_test(
"//mediapipe/framework:calculator_runner",
"//mediapipe/framework/port:gtest_main",
"@org_tensorflow//tensorflow/core:framework",
"@org_tensorflow//tensorflow/core:protos_all",
"@org_tensorflow//tensorflow/core:protos_all_cc",
],
)
cc_test(
name = "vector_float_to_tensor_calculator_test",
srcs = ["vector_float_to_tensor_calculator_test.cc"],
linkstatic = 1,
deps = [
":vector_float_to_tensor_calculator",
":vector_float_to_tensor_calculator_options_cc_proto",
@@ -1058,7 +1068,7 @@ cc_test(
"//mediapipe/framework:calculator_runner",
"//mediapipe/framework/port:gtest_main",
"@org_tensorflow//tensorflow/core:framework",
"@org_tensorflow//tensorflow/core:protos_all",
"@org_tensorflow//tensorflow/core:protos_all_cc",
],
)
mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator.cc
+2 -2
View File
@@ -17,7 +17,7 @@
#if !defined(__ANDROID__)
#include "mediapipe/framework/port/file_helpers.h"
#endif
#include "absl/strings/substitute.h"
#include "absl/strings/str_replace.h"
#include "mediapipe/calculators/tensorflow/tensorflow_session.h"
#include "mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
@@ -63,7 +63,7 @@ const std::string MaybeConvertSignatureToTag(
output.resize(name.length());
std::transform(name.begin(), name.end(), output.begin(),
[](unsigned char c) { return std::toupper(c); });
output = absl::Substitute(output, "/", "_");
output = absl::StrReplaceAll(output, {{"/", "_"}});
return output;
} else {
return name;
mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator_test.cc
+1 -1
View File
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include "absl/strings/substitute.h"
#include "absl/strings/str_replace.h"
#include "mediapipe/calculators/tensorflow/tensorflow_session.h"
#include "mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator.pb.h"
#include "mediapipe/framework/calculator.pb.h"
mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator.cc
+2 -2
View File
@@ -17,7 +17,7 @@
#if !defined(__ANDROID__)
#include "mediapipe/framework/port/file_helpers.h"
#endif
#include "absl/strings/substitute.h"
#include "absl/strings/str_replace.h"
#include "mediapipe/calculators/tensorflow/tensorflow_session.h"
#include "mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator.pb.h"
#include "mediapipe/framework/deps/file_path.h"
@@ -65,7 +65,7 @@ const std::string MaybeConvertSignatureToTag(
output.resize(name.length());
std::transform(name.begin(), name.end(), output.begin(),
[](unsigned char c) { return std::toupper(c); });
output = absl::Substitute(output, "/", "_");
output = absl::StrReplaceAll(output, {{"/", "_"}});
return output;
} else {
return name;
mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator_test.cc
+1 -1
View File
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include "absl/strings/substitute.h"
#include "absl/strings/str_replace.h"
#include "mediapipe/calculators/tensorflow/tensorflow_session.h"
#include "mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
mediapipe/calculators/tflite/BUILD
+1
View File
@@ -485,6 +485,7 @@ cc_test(
"//mediapipe/framework/port:integral_types",
"//mediapipe/framework/port:parse_text_proto",
"//mediapipe/framework/tool:validate_type",
"@com_google_absl//absl/strings",
"@org_tensorflow//tensorflow/lite:framework",
"@org_tensorflow//tensorflow/lite/kernels:builtin_ops",
],
mediapipe/calculators/tflite/tflite_inference_calculator.cc
+72 -33
View File
@@ -148,7 +148,7 @@ struct GPUData {
// options: {
// [mediapipe.TfLiteInferenceCalculatorOptions.ext] {
// model_path: "modelname.tflite"
// use_gpu: true
// delegate { gpu {} }
// }
// }
// }
@@ -163,6 +163,9 @@ struct GPUData {
//
class TfLiteInferenceCalculator : public CalculatorBase {
public:
using TfLiteDelegatePtr =
std::unique_ptr<TfLiteDelegate, std::function<void(TfLiteDelegate*)>>;
static ::mediapipe::Status GetContract(CalculatorContract* cc);
::mediapipe::Status Open(CalculatorContext* cc) override;
@@ -176,7 +179,7 @@ class TfLiteInferenceCalculator : public CalculatorBase {
std::unique_ptr<tflite::Interpreter> interpreter_;
std::unique_ptr<tflite::FlatBufferModel> model_;
TfLiteDelegate* delegate_ = nullptr;
TfLiteDelegatePtr delegate_;
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
mediapipe::GlCalculatorHelper gpu_helper_;
@@ -212,12 +215,18 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
RET_CHECK(cc->Outputs().HasTag("TENSORS") ^
cc->Outputs().HasTag("TENSORS_GPU"));
bool use_gpu = false;
const auto& options =
cc->Options<::mediapipe::TfLiteInferenceCalculatorOptions>();
bool use_gpu =
options.has_delegate() ? options.delegate().has_gpu() : options.use_gpu();
if (cc->Inputs().HasTag("TENSORS"))
cc->Inputs().Tag("TENSORS").Set<std::vector<TfLiteTensor>>();
#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__)
if (cc->Inputs().HasTag("TENSORS_GPU")) {
RET_CHECK(!options.has_delegate() || options.delegate().has_gpu())
<< "GPU input is compatible with GPU delegate only.";
cc->Inputs().Tag("TENSORS_GPU").Set<std::vector<GpuTensor>>();
use_gpu |= true;
}
@@ -227,6 +236,9 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
cc->Outputs().Tag("TENSORS").Set<std::vector<TfLiteTensor>>();
#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__)
if (cc->Outputs().HasTag("TENSORS_GPU")) {
RET_CHECK(!options.has_delegate() || options.delegate().has_gpu())
<< "GPU output is compatible with GPU delegate only.";
cc->Outputs().Tag("TENSORS_GPU").Set<std::vector<GpuTensor>>();
use_gpu |= true;
}
@@ -238,10 +250,6 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
.Set<tflite::ops::builtin::BuiltinOpResolver>();
}
const auto& options =
cc->Options<::mediapipe::TfLiteInferenceCalculatorOptions>();
use_gpu |= options.use_gpu();
if (use_gpu) {
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc));
@@ -454,7 +462,7 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
if (gpu_inference_) {
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext([this]() -> Status {
TfLiteGpuDelegateDelete(delegate_);
delegate_ = nullptr;
for (int i = 0; i < gpu_data_in_.size(); ++i) {
gpu_data_in_[i].reset();
}
@@ -464,7 +472,7 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
return ::mediapipe::OkStatus();
}));
#elif defined(MEDIAPIPE_IOS)
TFLGpuDelegateDelete(delegate_);
delegate_ = nullptr;
for (int i = 0; i < gpu_data_in_.size(); ++i) {
gpu_data_in_[i].reset();
}
@@ -472,8 +480,9 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
gpu_data_out_[i].reset();
}
#endif
} else {
delegate_ = nullptr;
}
delegate_ = nullptr;
}
#if defined(MEDIAPIPE_EDGE_TPU)
edgetpu_context_.reset();
@@ -501,7 +510,8 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
}
// Get execution modes.
gpu_inference_ = options.use_gpu();
gpu_inference_ =
options.has_delegate() ? options.delegate().has_gpu() : options.use_gpu();
return ::mediapipe::OkStatus();
}
@@ -526,8 +536,12 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
RET_CHECK(interpreter_);
#if defined(__EMSCRIPTEN__)
#if defined(__EMSCRIPTEN__) || defined(MEDIAPIPE_EDGE_TPU)
interpreter_->SetNumThreads(1);
#else
interpreter_->SetNumThreads(
cc->Options<mediapipe::TfLiteInferenceCalculatorOptions>()
.cpu_num_thread());
#endif // __EMSCRIPTEN__
if (gpu_output_) {
@@ -545,20 +559,37 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
::mediapipe::Status TfLiteInferenceCalculator::LoadDelegate(
CalculatorContext* cc) {
#if defined(MEDIAPIPE_ANDROID)
const auto& calculator_opts =
cc->Options<mediapipe::TfLiteInferenceCalculatorOptions>();
if (calculator_opts.has_delegate() &&
calculator_opts.delegate().has_tflite()) {
// Default tflite inference requeqsted - no need to modify graph.
return ::mediapipe::OkStatus();
}
if (!gpu_inference_) {
if (cc->Options<mediapipe::TfLiteInferenceCalculatorOptions>()
.use_nnapi()) {
#if defined(MEDIAPIPE_ANDROID)
const bool nnapi_requested = calculator_opts.has_delegate()
? calculator_opts.delegate().has_nnapi()
: calculator_opts.use_nnapi();
if (nnapi_requested) {
// Attempt to use NNAPI.
// If not supported, the default CPU delegate will be created and used.
interpreter_->SetAllowFp16PrecisionForFp32(1);
delegate_ = tflite::NnApiDelegate();
RET_CHECK_EQ(interpreter_->ModifyGraphWithDelegate(delegate_), kTfLiteOk);
delegate_ =
TfLiteDelegatePtr(tflite::NnApiDelegate(), [](TfLiteDelegate*) {
// No need to free according to tflite::NnApiDelegate()
// documentation.
});
RET_CHECK_EQ(interpreter_->ModifyGraphWithDelegate(delegate_.get()),
kTfLiteOk);
return ::mediapipe::OkStatus();
}
#endif // MEDIAPIPE_ANDROID
// Return, no need for GPU delegate below.
return ::mediapipe::OkStatus();
}
#endif // ANDROID
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
// Configure and create the delegate.
@@ -568,7 +599,9 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
TFLITE_GL_OBJECT_TYPE_FASTEST;
options.compile_options.dynamic_batch_enabled = 0;
options.compile_options.inline_parameters = 1;
if (!delegate_) delegate_ = TfLiteGpuDelegateCreate(&options);
if (!delegate_)
delegate_ = TfLiteDelegatePtr(TfLiteGpuDelegateCreate(&options),
&TfLiteGpuDelegateDelete);
if (gpu_input_) {
// Get input image sizes.
@@ -586,7 +619,7 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
::tflite::gpu::gl::CreateReadWriteShaderStorageBuffer<float>(
gpu_data_in_[i]->elements, &gpu_data_in_[i]->buffer));
RET_CHECK_EQ(TfLiteGpuDelegateBindBufferToTensor(
delegate_, gpu_data_in_[i]->buffer.id(),
delegate_.get(), gpu_data_in_[i]->buffer.id(),
interpreter_->inputs()[i]),
kTfLiteOk);
}
@@ -609,15 +642,16 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
for (int i = 0; i < gpu_data_out_.size(); ++i) {
RET_CHECK_CALL(CreateReadWriteShaderStorageBuffer<float>(
gpu_data_out_[i]->elements, &gpu_data_out_[i]->buffer));
RET_CHECK_EQ(
TfLiteGpuDelegateBindBufferToTensor(
delegate_, gpu_data_out_[i]->buffer.id(), output_indices[i]),
kTfLiteOk);
RET_CHECK_EQ(TfLiteGpuDelegateBindBufferToTensor(
delegate_.get(), gpu_data_out_[i]->buffer.id(),
output_indices[i]),
kTfLiteOk);
}
}
// Must call this last.
RET_CHECK_EQ(interpreter_->ModifyGraphWithDelegate(delegate_), kTfLiteOk);
RET_CHECK_EQ(interpreter_->ModifyGraphWithDelegate(delegate_.get()),
kTfLiteOk);
#endif // OpenGL
#if defined(MEDIAPIPE_IOS)
@@ -626,7 +660,9 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
TFLGpuDelegateOptions options;
options.allow_precision_loss = true;
options.wait_type = TFLGpuDelegateWaitType::TFLGpuDelegateWaitTypePassive;
if (!delegate_) delegate_ = TFLGpuDelegateCreate(&options);
if (!delegate_)
delegate_ = TfLiteDelegatePtr(TFLGpuDelegateCreate(&options),
&TFLGpuDelegateDelete);
id<MTLDevice> device = gpu_helper_.mtlDevice;
if (gpu_input_) {
@@ -678,10 +714,12 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
gpu_data_in_[i]->buffer =
[device newBufferWithLength:gpu_data_in_[i]->elements * kHalfSize
options:MTLResourceStorageModeShared];
RET_CHECK_EQ(interpreter_->ModifyGraphWithDelegate(delegate_), kTfLiteOk);
RET_CHECK_EQ(TFLGpuDelegateBindMetalBufferToTensor(
delegate_, input_indices[i], gpu_data_in_[i]->buffer),
true);
RET_CHECK_EQ(interpreter_->ModifyGraphWithDelegate(delegate_.get()),
kTfLiteOk);
RET_CHECK_EQ(
TFLGpuDelegateBindMetalBufferToTensor(
delegate_.get(), input_indices[i], gpu_data_in_[i]->buffer),
true);
}
}
if (gpu_output_) {
@@ -725,9 +763,10 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
gpu_data_out_[i]->buffer =
[device newBufferWithLength:gpu_data_out_[i]->elements * kHalfSize
options:MTLResourceStorageModeShared];
RET_CHECK_EQ(TFLGpuDelegateBindMetalBufferToTensor(
delegate_, output_indices[i], gpu_data_out_[i]->buffer),
true);
RET_CHECK_EQ(
TFLGpuDelegateBindMetalBufferToTensor(
delegate_.get(), output_indices[i], gpu_data_out_[i]->buffer),
true);
}
// Create converter for GPU output.
mediapipe/calculators/tflite/tflite_inference_calculator.proto
+31 -3
View File
@@ -27,7 +27,7 @@ import "mediapipe/framework/calculator.proto";
// options {
// [mediapipe.TfLiteInferenceCalculatorOptions.ext] {
// model_path: "model.tflite"
// use_gpu: true
// delegate { gpu {} }
// }
// }
// }
@@ -37,6 +37,22 @@ message TfLiteInferenceCalculatorOptions {
optional TfLiteInferenceCalculatorOptions ext = 233867213;
}
message Delegate {
// Default inference provided by tflite.
message TfLite {}
// Delegate to run GPU inference depending on the device.
// (Can use OpenGl, OpenCl, Metal depending on the device.)
message Gpu {}
// Android only.
message Nnapi {}
oneof delegate {
TfLite tflite = 1;
Gpu gpu = 2;
Nnapi nnapi = 3;
}
}
// Path to the TF Lite model (ex: /path/to/modelname.tflite).
// On mobile, this is generally just modelname.tflite.
optional string model_path = 1;
@@ -44,10 +60,22 @@ message TfLiteInferenceCalculatorOptions {
// Whether the TF Lite GPU or CPU backend should be used. Effective only when
// input tensors are on CPU. For input tensors on GPU, GPU backend is always
// used.
optional bool use_gpu = 2 [default = false];
// DEPRECATED: configure "delegate" instead.
optional bool use_gpu = 2 [deprecated = true, default = false];
// Android only. When true, an NNAPI delegate will be used for inference.
// If NNAPI is not available, then the default CPU delegate will be used
// automatically.
optional bool use_nnapi = 3 [default = false];
// DEPRECATED: configure "delegate" instead.
optional bool use_nnapi = 3 [deprecated = true, default = false];
// The number of threads available to the interpreter. Effective only when
// input tensors are on CPU and 'use_gpu' is false.
optional int32 cpu_num_thread = 4 [default = -1];
// TfLite delegate to run inference.
// NOTE: calculator is free to choose delegate if not specified explicitly.
// NOTE: use_gpu/use_nnapi are ignored if specified. (Delegate takes
// precedence over use_* deprecated options.)
optional Delegate delegate = 5;
}
mediapipe/calculators/tflite/tflite_inference_calculator_test.cc
+25 -22
View File
@@ -16,6 +16,8 @@
#include <string>
#include <vector>
#include "absl/strings/str_replace.h"
#include "absl/strings/string_view.h"
#include "mediapipe/calculators/tflite/tflite_inference_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_runner.h"
@@ -39,13 +41,7 @@ namespace mediapipe {
using ::tflite::Interpreter;
class TfLiteInferenceCalculatorTest : public ::testing::Test {
protected:
std::unique_ptr<CalculatorRunner> runner_ = nullptr;
};
// Tests a simple add model that adds an input tensor to itself.
TEST_F(TfLiteInferenceCalculatorTest, SmokeTest) {
void DoSmokeTest(absl::string_view delegate) {
const int width = 8;
const int height = 8;
const int channels = 3;
@@ -73,23 +69,24 @@ TEST_F(TfLiteInferenceCalculatorTest, SmokeTest) {
auto input_vec = absl::make_unique<std::vector<TfLiteTensor>>();
input_vec->emplace_back(*tensor);
std::string graph_proto = R"(
input_stream: "tensor_in"
node {
calculator: "TfLiteInferenceCalculator"
input_stream: "TENSORS:tensor_in"
output_stream: "TENSORS:tensor_out"
options {
[mediapipe.TfLiteInferenceCalculatorOptions.ext] {
model_path: "mediapipe/calculators/tflite/testdata/add.bin"
$delegate
}
}
}
)";
ASSERT_EQ(absl::StrReplaceAll({{"$delegate", delegate}}, &graph_proto), 1);
// Prepare single calculator graph to and wait for packets.
CalculatorGraphConfig graph_config =
::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
R"(
input_stream: "tensor_in"
node {
calculator: "TfLiteInferenceCalculator"
input_stream: "TENSORS:tensor_in"
output_stream: "TENSORS:tensor_out"
options {
[mediapipe.TfLiteInferenceCalculatorOptions.ext] {
use_gpu: false
model_path: "mediapipe/calculators/tflite/testdata/add.bin"
}
}
}
)");
::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(graph_proto);
std::vector<Packet> output_packets;
tool::AddVectorSink("tensor_out", &graph_config, &output_packets);
CalculatorGraph graph(graph_config);
@@ -120,4 +117,10 @@ TEST_F(TfLiteInferenceCalculatorTest, SmokeTest) {
MP_ASSERT_OK(graph.WaitUntilDone());
}
// Tests a simple add model that adds an input tensor to itself.
TEST(TfLiteInferenceCalculatorTest, SmokeTest) {
DoSmokeTest(/*delegate=*/"");
DoSmokeTest(/*delegate=*/"delegate { tflite {} }");
}
} // namespace mediapipe
mediapipe/calculators/tflite/tflite_tensors_to_landmarks_calculator.cc
+59 -3
View File
@@ -28,6 +28,21 @@ namespace mediapipe {
// TENSORS - Vector of TfLiteTensor of type kTfLiteFloat32. Only the first
// tensor will be used. The size of the values must be
// (num_dimension x num_landmarks).
//
// FLIP_HORIZONTALLY (optional): Whether to flip landmarks horizontally or
// not. Overrides corresponding side packet and/or field in the calculator
// options.
//
// FLIP_VERTICALLY (optional): Whether to flip landmarks vertically or not.
// Overrides corresponding side packet and/or field in the calculator options.
//
// Input side packet:
// FLIP_HORIZONTALLY (optional): Whether to flip landmarks horizontally or
// not. Overrides the corresponding field in the calculator options.
//
// FLIP_VERTICALLY (optional): Whether to flip landmarks vertically or not.
// Overrides the corresponding field in the calculator options.
//
// Output:
// LANDMARKS(optional) - Result MediaPipe landmarks.
// NORM_LANDMARKS(optional) - Result MediaPipe normalized landmarks.
@@ -61,6 +76,8 @@ class TfLiteTensorsToLandmarksCalculator : public CalculatorBase {
private:
::mediapipe::Status LoadOptions(CalculatorContext* cc);
int num_landmarks_ = 0;
bool flip_vertically_ = false;
bool flip_horizontally_ = false;
::mediapipe::TfLiteTensorsToLandmarksCalculatorOptions options_;
};
@@ -75,6 +92,22 @@ REGISTER_CALCULATOR(TfLiteTensorsToLandmarksCalculator);
cc->Inputs().Tag("TENSORS").Set<std::vector<TfLiteTensor>>();
}
if (cc->Inputs().HasTag("FLIP_HORIZONTALLY")) {
cc->Inputs().Tag("FLIP_HORIZONTALLY").Set<bool>();
}
if (cc->Inputs().HasTag("FLIP_VERTICALLY")) {
cc->Inputs().Tag("FLIP_VERTICALLY").Set<bool>();
}
if (cc->InputSidePackets().HasTag("FLIP_HORIZONTALLY")) {
cc->InputSidePackets().Tag("FLIP_HORIZONTALLY").Set<bool>();
}
if (cc->InputSidePackets().HasTag("FLIP_VERTICALLY")) {
cc->InputSidePackets().Tag("FLIP_VERTICALLY").Set<bool>();
}
if (cc->Outputs().HasTag("LANDMARKS")) {
cc->Outputs().Tag("LANDMARKS").Set<LandmarkList>();
}
@@ -98,17 +131,40 @@ REGISTER_CALCULATOR(TfLiteTensorsToLandmarksCalculator);
<< "Must provide input with/height for getting normalized landmarks.";
}
if (cc->Outputs().HasTag("LANDMARKS") &&
(options_.flip_vertically() || options_.flip_horizontally())) {
(options_.flip_vertically() || options_.flip_horizontally() ||
cc->InputSidePackets().HasTag("FLIP_HORIZONTALLY") ||
cc->InputSidePackets().HasTag("FLIP_VERTICALLY"))) {
RET_CHECK(options_.has_input_image_height() &&
options_.has_input_image_width())
<< "Must provide input with/height for using flip_vertically option "
"when outputing landmarks in absolute coordinates.";
}
flip_horizontally_ =
cc->InputSidePackets().HasTag("FLIP_HORIZONTALLY")
? cc->InputSidePackets().Tag("FLIP_HORIZONTALLY").Get<bool>()
: options_.flip_horizontally();
flip_horizontally_ =
cc->InputSidePackets().HasTag("FLIP_VERTICALLY")
? cc->InputSidePackets().Tag("FLIP_VERTICALLY").Get<bool>()
: options_.flip_vertically();
return ::mediapipe::OkStatus();
}
::mediapipe::Status TfLiteTensorsToLandmarksCalculator::Process(
CalculatorContext* cc) {
// Override values if specified so.
if (cc->Inputs().HasTag("FLIP_HORIZONTALLY") &&
!cc->Inputs().Tag("FLIP_HORIZONTALLY").IsEmpty()) {
flip_horizontally_ = cc->Inputs().Tag("FLIP_HORIZONTALLY").Get<bool>();
}
if (cc->Inputs().HasTag("FLIP_VERTICALLY") &&
!cc->Inputs().Tag("FLIP_VERTICALLY").IsEmpty()) {
flip_vertically_ = cc->Inputs().Tag("FLIP_VERTICALLY").Get<bool>();
}
if (cc->Inputs().Tag("TENSORS").IsEmpty()) {
return ::mediapipe::OkStatus();
}
@@ -133,13 +189,13 @@ REGISTER_CALCULATOR(TfLiteTensorsToLandmarksCalculator);
const int offset = ld * num_dimensions;
Landmark* landmark = output_landmarks.add_landmark();
if (options_.flip_horizontally()) {
if (flip_horizontally_) {
landmark->set_x(options_.input_image_width() - raw_landmarks[offset]);
} else {
landmark->set_x(raw_landmarks[offset]);
}
if (num_dimensions > 1) {
if (options_.flip_vertically()) {
if (flip_vertically_) {
landmark->set_y(options_.input_image_height() -
raw_landmarks[offset + 1]);
} else {
mediapipe/calculators/util/BUILD
+14
View File
@@ -437,6 +437,7 @@ cc_library(
"//mediapipe/framework/formats:rect_cc_proto",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"@com_google_absl//absl/types:optional",
],
alwayslink = 1,
)
@@ -928,6 +929,19 @@ cc_library(
alwayslink = 1,
)
cc_library(
name = "local_file_pattern_contents_calculator",
srcs = ["local_file_pattern_contents_calculator.cc"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/port:file_helpers",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
],
alwayslink = 1,
)
cc_library(
name = "filter_collection_calculator",
srcs = ["filter_collection_calculator.cc"],
mediapipe/calculators/util/annotation_overlay_calculator.cc
+10 -10
View File
@@ -39,13 +39,13 @@ namespace mediapipe {
namespace {
constexpr char kInputFrameTag[] = "INPUT_FRAME";
constexpr char kOutputFrameTag[] = "OUTPUT_FRAME";
constexpr char kInputFrameTag[] = "IMAGE";
constexpr char kOutputFrameTag[] = "IMAGE";
constexpr char kInputVectorTag[] = "VECTOR";
constexpr char kInputFrameTagGpu[] = "INPUT_FRAME_GPU";
constexpr char kOutputFrameTagGpu[] = "OUTPUT_FRAME_GPU";
constexpr char kInputFrameTagGpu[] = "IMAGE_GPU";
constexpr char kOutputFrameTagGpu[] = "IMAGE_GPU";
enum { ATTRIB_VERTEX, ATTRIB_TEXTURE_POSITION, NUM_ATTRIBUTES };
@@ -61,7 +61,7 @@ constexpr int kAnnotationBackgroundColor[] = {100, 101, 102};
// A calculator for rendering data on images.
//
// Inputs:
// 1. INPUT_FRAME or INPUT_FRAME_GPU (optional): An ImageFrame (or GpuBuffer)
// 1. IMAGE or IMAGE_GPU (optional): An ImageFrame (or GpuBuffer)
// containing the input image.
// If output is CPU, and input isn't provided, the renderer creates a
// blank canvas with the width, height and color provided in the options.
@@ -73,7 +73,7 @@ constexpr int kAnnotationBackgroundColor[] = {100, 101, 102};
// input vector items. These input streams are tagged with "VECTOR".
//
// Output:
// 1. OUTPUT_FRAME or OUTPUT_FRAME_GPU: A rendered ImageFrame (or GpuBuffer).
// 1. IMAGE or IMAGE_GPU: A rendered ImageFrame (or GpuBuffer).
//
// For CPU input frames, only SRGBA, SRGB and GRAY8 format are supported. The
// output format is the same as input except for GRAY8 where the output is in
@@ -87,13 +87,13 @@ constexpr int kAnnotationBackgroundColor[] = {100, 101, 102};
// Example config (CPU):
// node {
// calculator: "AnnotationOverlayCalculator"
// input_stream: "INPUT_FRAME:image_frames"
// input_stream: "IMAGE:image_frames"
// input_stream: "render_data_1"
// input_stream: "render_data_2"
// input_stream: "render_data_3"
// input_stream: "VECTOR:0:render_data_vec_0"
// input_stream: "VECTOR:1:render_data_vec_1"
// output_stream: "OUTPUT_FRAME:decorated_frames"
// output_stream: "IMAGE:decorated_frames"
// options {
// [mediapipe.AnnotationOverlayCalculatorOptions.ext] {
// }
@@ -103,13 +103,13 @@ constexpr int kAnnotationBackgroundColor[] = {100, 101, 102};
// Example config (GPU):
// node {
// calculator: "AnnotationOverlayCalculator"
// input_stream: "INPUT_FRAME_GPU:image_frames"
// input_stream: "IMAGE_GPU:image_frames"
// input_stream: "render_data_1"
// input_stream: "render_data_2"
// input_stream: "render_data_3"
// input_stream: "VECTOR:0:render_data_vec_0"
// input_stream: "VECTOR:1:render_data_vec_1"
// output_stream: "OUTPUT_FRAME_GPU:decorated_frames"
// output_stream: "IMAGE_GPU:decorated_frames"
// options {
// [mediapipe.AnnotationOverlayCalculatorOptions.ext] {
// }
mediapipe/calculators/util/detections_to_rects_calculator.cc
+52 -28
View File
@@ -39,7 +39,8 @@ constexpr char kNormRectsTag[] = "NORM_RECTS";
} // namespace
::mediapipe::Status DetectionsToRectsCalculator::DetectionToRect(
const Detection& detection, Rect* rect) {
const Detection& detection, const DetectionSpec& detection_spec,
Rect* rect) {
const LocationData location_data = detection.location_data();
RET_CHECK(location_data.format() == LocationData::BOUNDING_BOX)
<< "Only Detection with formats of BOUNDING_BOX can be converted to Rect";
@@ -52,7 +53,8 @@ constexpr char kNormRectsTag[] = "NORM_RECTS";
}
::mediapipe::Status DetectionsToRectsCalculator::DetectionToNormalizedRect(
const Detection& detection, NormalizedRect* rect) {
const Detection& detection, const DetectionSpec& detection_spec,
NormalizedRect* rect) {
const LocationData location_data = detection.location_data();
RET_CHECK(location_data.format() == LocationData::RELATIVE_BOUNDING_BOX)
<< "Only Detection with formats of RELATIVE_BOUNDING_BOX can be "
@@ -174,27 +176,31 @@ constexpr char kNormRectsTag[] = "NORM_RECTS";
}
}
std::pair<int, int> image_size;
if (rotate_) {
RET_CHECK(!cc->Inputs().Tag(kImageSizeTag).IsEmpty());
image_size = cc->Inputs().Tag(kImageSizeTag).Get<std::pair<int, int>>();
}
// Get dynamic calculator options (e.g. `image_size`).
const DetectionSpec detection_spec = GetDetectionSpec(cc);
if (cc->Outputs().HasTag(kRectTag)) {
auto output_rect = absl::make_unique<Rect>();
MP_RETURN_IF_ERROR(DetectionToRect(detections[0], output_rect.get()));
MP_RETURN_IF_ERROR(
DetectionToRect(detections[0], detection_spec, output_rect.get()));
if (rotate_) {
output_rect->set_rotation(ComputeRotation(detections[0], image_size));
float rotation;
MP_RETURN_IF_ERROR(
ComputeRotation(detections[0], detection_spec, &rotation));
output_rect->set_rotation(rotation);
}
cc->Outputs().Tag(kRectTag).Add(output_rect.release(),
cc->InputTimestamp());
}
if (cc->Outputs().HasTag(kNormRectTag)) {
auto output_rect = absl::make_unique<NormalizedRect>();
MP_RETURN_IF_ERROR(
DetectionToNormalizedRect(detections[0], output_rect.get()));
MP_RETURN_IF_ERROR(DetectionToNormalizedRect(detections[0], detection_spec,
output_rect.get()));
if (rotate_) {
output_rect->set_rotation(ComputeRotation(detections[0], image_size));
float rotation;
MP_RETURN_IF_ERROR(
ComputeRotation(detections[0], detection_spec, &rotation));
output_rect->set_rotation(rotation);
}
cc->Outputs()
.Tag(kNormRectTag)
@@ -203,11 +209,13 @@ constexpr char kNormRectsTag[] = "NORM_RECTS";
if (cc->Outputs().HasTag(kRectsTag)) {
auto output_rects = absl::make_unique<std::vector<Rect>>(detections.size());
for (int i = 0; i < detections.size(); ++i) {
MP_RETURN_IF_ERROR(
DetectionToRect(detections[i], &(output_rects->at(i))));
MP_RETURN_IF_ERROR(DetectionToRect(detections[i], detection_spec,
&(output_rects->at(i))));
if (rotate_) {
output_rects->at(i).set_rotation(
ComputeRotation(detections[i], image_size));
float rotation;
MP_RETURN_IF_ERROR(
ComputeRotation(detections[i], detection_spec, &rotation));
output_rects->at(i).set_rotation(rotation);
}
}
cc->Outputs().Tag(kRectsTag).Add(output_rects.release(),
@@ -217,11 +225,13 @@ constexpr char kNormRectsTag[] = "NORM_RECTS";
auto output_rects =
absl::make_unique<std::vector<NormalizedRect>>(detections.size());
for (int i = 0; i < detections.size(); ++i) {
MP_RETURN_IF_ERROR(
DetectionToNormalizedRect(detections[i], &(output_rects->at(i))));
MP_RETURN_IF_ERROR(DetectionToNormalizedRect(
detections[i], detection_spec, &(output_rects->at(i))));
if (rotate_) {
output_rects->at(i).set_rotation(
ComputeRotation(detections[i], image_size));
float rotation;
MP_RETURN_IF_ERROR(
ComputeRotation(detections[i], detection_spec, &rotation));
output_rects->at(i).set_rotation(rotation);
}
}
cc->Outputs()
@@ -232,21 +242,35 @@ constexpr char kNormRectsTag[] = "NORM_RECTS";
return ::mediapipe::OkStatus();
}
float DetectionsToRectsCalculator::ComputeRotation(
const Detection& detection, const std::pair<int, int> image_size) {
::mediapipe::Status DetectionsToRectsCalculator::ComputeRotation(
const Detection& detection, const DetectionSpec& detection_spec,
float* rotation) {
const auto& location_data = detection.location_data();
const auto& image_size = detection_spec.image_size;
RET_CHECK(image_size) << "Image size is required to calculate rotation";
const float x0 = location_data.relative_keypoints(start_keypoint_index_).x() *
image_size.first;
image_size->first;
const float y0 = location_data.relative_keypoints(start_keypoint_index_).y() *
image_size.second;
image_size->second;
const float x1 = location_data.relative_keypoints(end_keypoint_index_).x() *
image_size.first;
image_size->first;
const float y1 = location_data.relative_keypoints(end_keypoint_index_).y() *
image_size.second;
image_size->second;
float rotation = target_angle_ - std::atan2(-(y1 - y0), x1 - x0);
*rotation = NormalizeRadians(target_angle_ - std::atan2(-(y1 - y0), x1 - x0));
return NormalizeRadians(rotation);
return ::mediapipe::OkStatus();
}
DetectionSpec DetectionsToRectsCalculator::GetDetectionSpec(
const CalculatorContext* cc) {
absl::optional<std::pair<int, int>> image_size;
if (cc->Inputs().HasTag(kImageSizeTag)) {
image_size = cc->Inputs().Tag(kImageSizeTag).Get<std::pair<int, int>>();
}
return {image_size};
}
REGISTER_CALCULATOR(DetectionsToRectsCalculator);
mediapipe/calculators/util/detections_to_rects_calculator.h
+15 -4
View File
@@ -16,6 +16,7 @@
#include <cmath>
#include "absl/types/optional.h"
#include "mediapipe/calculators/util/detections_to_rects_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_options.pb.h"
@@ -27,6 +28,13 @@
namespace mediapipe {
// Dynamic options passed as calculator `input_stream` that can be used for
// calculation of rectangle or rotation for given detection. Does not include
// static calculator options which are available via private fields.
struct DetectionSpec {
absl::optional<std::pair<int, int>> image_size;
};
// A calculator that converts Detection proto to Rect proto.
//
// Detection is the format for encoding one or more detections in an image.
@@ -81,13 +89,16 @@ class DetectionsToRectsCalculator : public CalculatorBase {
::mediapipe::Status Process(CalculatorContext* cc) override;
protected:
virtual float ComputeRotation(const ::mediapipe::Detection& detection,
const std::pair<int, int> image_size);
virtual ::mediapipe::Status DetectionToRect(
const ::mediapipe::Detection& detection, ::mediapipe::Rect* rect);
const ::mediapipe::Detection& detection,
const DetectionSpec& detection_spec, ::mediapipe::Rect* rect);
virtual ::mediapipe::Status DetectionToNormalizedRect(
const ::mediapipe::Detection& detection,
::mediapipe::NormalizedRect* rect);
const DetectionSpec& detection_spec, ::mediapipe::NormalizedRect* rect);
virtual ::mediapipe::Status ComputeRotation(
const ::mediapipe::Detection& detection,
const DetectionSpec& detection_spec, float* rotation);
virtual DetectionSpec GetDetectionSpec(const CalculatorContext* cc);
static inline float NormalizeRadians(float angle) {
return angle - 2 * M_PI * std::floor((angle - (-M_PI)) / (2 * M_PI));
mediapipe/calculators/util/landmark_letterbox_removal_calculator.cc
+49 -37
View File
@@ -12,20 +12,6 @@
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <cmath>
#include <vector>
@@ -67,6 +53,15 @@ constexpr char kLetterboxPaddingTag[] = "LETTERBOX_PADDING";
// input_stream: "LETTERBOX_PADDING:letterbox_padding"
// output_stream: "LANDMARKS:adjusted_landmarks"
// }
//
// node {
// calculator: "LandmarkLetterboxRemovalCalculator"
// input_stream: "LANDMARKS:0:landmarks_0"
// input_stream: "LANDMARKS:1:landmarks_1"
// input_stream: "LETTERBOX_PADDING:letterbox_padding"
// output_stream: "LANDMARKS:0:adjusted_landmarks_0"
// output_stream: "LANDMARKS:1:adjusted_landmarks_1"
// }
class LandmarkLetterboxRemovalCalculator : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
@@ -74,10 +69,20 @@ class LandmarkLetterboxRemovalCalculator : public CalculatorBase {
cc->Inputs().HasTag(kLetterboxPaddingTag))
<< "Missing one or more input streams.";
cc->Inputs().Tag(kLandmarksTag).Set<NormalizedLandmarkList>();
RET_CHECK_EQ(cc->Inputs().NumEntries(kLandmarksTag),
cc->Outputs().NumEntries(kLandmarksTag))
<< "Same number of input and output landmarks is required.";
for (CollectionItemId id = cc->Inputs().BeginId(kLandmarksTag);
id != cc->Inputs().EndId(kLandmarksTag); ++id) {
cc->Inputs().Get(id).Set<NormalizedLandmarkList>();
}
cc->Inputs().Tag(kLetterboxPaddingTag).Set<std::array<float, 4>>();
cc->Outputs().Tag(kLandmarksTag).Set<NormalizedLandmarkList>();
for (CollectionItemId id = cc->Outputs().BeginId(kLandmarksTag);
id != cc->Outputs().EndId(kLandmarksTag); ++id) {
cc->Outputs().Get(id).Set<NormalizedLandmarkList>();
}
return ::mediapipe::OkStatus();
}
@@ -89,38 +94,45 @@ class LandmarkLetterboxRemovalCalculator : public CalculatorBase {
}
::mediapipe::Status Process(CalculatorContext* cc) override {
// Only process if there's input landmarks.
if (cc->Inputs().Tag(kLandmarksTag).IsEmpty()) {
if (cc->Inputs().Tag(kLetterboxPaddingTag).IsEmpty()) {
return ::mediapipe::OkStatus();
}
const NormalizedLandmarkList& input_landmarks =
cc->Inputs().Tag(kLandmarksTag).Get<NormalizedLandmarkList>();
const auto& letterbox_padding =
cc->Inputs().Tag(kLetterboxPaddingTag).Get<std::array<float, 4>>();
const float left = letterbox_padding[0];
const float top = letterbox_padding[1];
const float left_and_right = letterbox_padding[0] + letterbox_padding[2];
const float top_and_bottom = letterbox_padding[1] + letterbox_padding[3];
NormalizedLandmarkList output_landmarks;
for (int i = 0; i < input_landmarks.landmark_size(); ++i) {
const NormalizedLandmark& landmark = input_landmarks.landmark(i);
NormalizedLandmark* new_landmark = output_landmarks.add_landmark();
const float new_x = (landmark.x() - left) / (1.0f - left_and_right);
const float new_y = (landmark.y() - top) / (1.0f - top_and_bottom);
CollectionItemId input_id = cc->Inputs().BeginId(kLandmarksTag);
CollectionItemId output_id = cc->Outputs().BeginId(kLandmarksTag);
// Number of inputs and outpus is the same according to the contract.
for (; input_id != cc->Inputs().EndId(kLandmarksTag);
++input_id, ++output_id) {
const auto& input_packet = cc->Inputs().Get(input_id);
if (input_packet.IsEmpty()) {
continue;
}
new_landmark->set_x(new_x);
new_landmark->set_y(new_y);
// Keep z-coord as is.
new_landmark->set_z(landmark.z());
const NormalizedLandmarkList& input_landmarks =
input_packet.Get<NormalizedLandmarkList>();
NormalizedLandmarkList output_landmarks;
for (int i = 0; i < input_landmarks.landmark_size(); ++i) {
const NormalizedLandmark& landmark = input_landmarks.landmark(i);
NormalizedLandmark* new_landmark = output_landmarks.add_landmark();
const float new_x = (landmark.x() - left) / (1.0f - left_and_right);
const float new_y = (landmark.y() - top) / (1.0f - top_and_bottom);
new_landmark->set_x(new_x);
new_landmark->set_y(new_y);
// Keep z-coord as is.
new_landmark->set_z(landmark.z());
}
cc->Outputs().Get(output_id).AddPacket(
MakePacket<NormalizedLandmarkList>(output_landmarks)
.At(cc->InputTimestamp()));
}
cc->Outputs()
.Tag(kLandmarksTag)
.AddPacket(MakePacket<NormalizedLandmarkList>(output_landmarks)
.At(cc->InputTimestamp()));
return ::mediapipe::OkStatus();
}
};
mediapipe/calculators/util/landmark_projection_calculator.cc
+57 -43
View File
@@ -12,20 +12,6 @@
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <cmath>
#include <vector>
@@ -63,6 +49,15 @@ constexpr char kRectTag[] = "NORM_RECT";
// input_stream: "NORM_RECT:rect"
// output_stream: "NORM_LANDMARKS:projected_landmarks"
// }
//
// node {
// calculator: "LandmarkProjectionCalculator"
// input_stream: "NORM_LANDMARKS:0:landmarks_0"
// input_stream: "NORM_LANDMARKS:1:landmarks_1"
// input_stream: "NORM_RECT:rect"
// output_stream: "NORM_LANDMARKS:0:projected_landmarks_0"
// output_stream: "NORM_LANDMARKS:1:projected_landmarks_1"
// }
class LandmarkProjectionCalculator : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
@@ -70,10 +65,20 @@ class LandmarkProjectionCalculator : public CalculatorBase {
cc->Inputs().HasTag(kRectTag))
<< "Missing one or more input streams.";
cc->Inputs().Tag(kLandmarksTag).Set<NormalizedLandmarkList>();
RET_CHECK_EQ(cc->Inputs().NumEntries(kLandmarksTag),
cc->Outputs().NumEntries(kLandmarksTag))
<< "Same number of input and output landmarks is required.";
for (CollectionItemId id = cc->Inputs().BeginId(kLandmarksTag);
id != cc->Inputs().EndId(kLandmarksTag); ++id) {
cc->Inputs().Get(id).Set<NormalizedLandmarkList>();
}
cc->Inputs().Tag(kRectTag).Set<NormalizedRect>();
cc->Outputs().Tag(kLandmarksTag).Set<NormalizedLandmarkList>();
for (CollectionItemId id = cc->Outputs().BeginId(kLandmarksTag);
id != cc->Outputs().EndId(kLandmarksTag); ++id) {
cc->Outputs().Get(id).Set<NormalizedLandmarkList>();
}
return ::mediapipe::OkStatus();
}
@@ -85,41 +90,50 @@ class LandmarkProjectionCalculator : public CalculatorBase {
}
::mediapipe::Status Process(CalculatorContext* cc) override {
const auto& options =
cc->Options<::mediapipe::LandmarkProjectionCalculatorOptions>();
// Only process if there's input landmarks.
if (cc->Inputs().Tag(kLandmarksTag).IsEmpty()) {
if (cc->Inputs().Tag(kRectTag).IsEmpty()) {
return ::mediapipe::OkStatus();
}
const NormalizedLandmarkList& input_landmarks =
cc->Inputs().Tag(kLandmarksTag).Get<NormalizedLandmarkList>();
const auto& input_rect = cc->Inputs().Tag(kRectTag).Get<NormalizedRect>();
NormalizedLandmarkList output_landmarks;
for (int i = 0; i < input_landmarks.landmark_size(); ++i) {
const NormalizedLandmark& landmark = input_landmarks.landmark(i);
NormalizedLandmark* new_landmark = output_landmarks.add_landmark();
const auto& options =
cc->Options<::mediapipe::LandmarkProjectionCalculatorOptions>();
const float x = landmark.x() - 0.5f;
const float y = landmark.y() - 0.5f;
const float angle = options.ignore_rotation() ? 0 : input_rect.rotation();
float new_x = std::cos(angle) * x - std::sin(angle) * y;
float new_y = std::sin(angle) * x + std::cos(angle) * y;
CollectionItemId input_id = cc->Inputs().BeginId(kLandmarksTag);
CollectionItemId output_id = cc->Outputs().BeginId(kLandmarksTag);
// Number of inputs and outpus is the same according to the contract.
for (; input_id != cc->Inputs().EndId(kLandmarksTag);
++input_id, ++output_id) {
const auto& input_packet = cc->Inputs().Get(input_id);
if (input_packet.IsEmpty()) {
continue;
}
new_x = new_x * input_rect.width() + input_rect.x_center();
new_y = new_y * input_rect.height() + input_rect.y_center();
const auto& input_landmarks = input_packet.Get<NormalizedLandmarkList>();
NormalizedLandmarkList output_landmarks;
for (int i = 0; i < input_landmarks.landmark_size(); ++i) {
const NormalizedLandmark& landmark = input_landmarks.landmark(i);
NormalizedLandmark* new_landmark = output_landmarks.add_landmark();
new_landmark->set_x(new_x);
new_landmark->set_y(new_y);
// Keep z-coord as is.
new_landmark->set_z(landmark.z());
const float x = landmark.x() - 0.5f;
const float y = landmark.y() - 0.5f;
const float angle =
options.ignore_rotation() ? 0 : input_rect.rotation();
float new_x = std::cos(angle) * x - std::sin(angle) * y;
float new_y = std::sin(angle) * x + std::cos(angle) * y;
new_x = new_x * input_rect.width() + input_rect.x_center();
new_y = new_y * input_rect.height() + input_rect.y_center();
new_landmark->set_x(new_x);
new_landmark->set_y(new_y);
// Keep z-coord as is.
new_landmark->set_z(landmark.z());
}
cc->Outputs().Get(output_id).AddPacket(
MakePacket<NormalizedLandmarkList>(output_landmarks)
.At(cc->InputTimestamp()));
}
cc->Outputs()
.Tag(kLandmarksTag)
.AddPacket(MakePacket<NormalizedLandmarkList>(output_landmarks)
.At(cc->InputTimestamp()));
return ::mediapipe::OkStatus();
}
};
mediapipe/calculators/util/local_file_pattern_contents_calculator.cc
+75
View File
@@ -0,0 +1,75 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <memory>
#include <string>
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/port/file_helpers.h"
#include "mediapipe/framework/port/status.h"
namespace mediapipe {
// The calculator takes the path to local directory and desired file suffix to
// mach as input side packets, and outputs the contents of those files that
// match the pattern. Those matched files will be sent sequentially through the
// output stream with incremental timestamp difference by 1.
//
// Example config:
// node {
// calculator: "LocalFilePatternContentsCalculator"
// input_side_packet: "FILE_DIRECTORY:file_directory"
// input_side_packet: "FILE_SUFFIX:file_suffix"
// output_stream: "CONTENTS:contents"
// }
class LocalFilePatternContentsCalculator : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
cc->InputSidePackets().Tag("FILE_DIRECTORY").Set<std::string>();
cc->InputSidePackets().Tag("FILE_SUFFIX").Set<std::string>();
cc->Outputs().Tag("CONTENTS").Set<std::string>();
return ::mediapipe::OkStatus();
}
::mediapipe::Status Open(CalculatorContext* cc) override {
MP_RETURN_IF_ERROR(::mediapipe::file::MatchFileTypeInDirectory(
cc->InputSidePackets().Tag("FILE_DIRECTORY").Get<std::string>(),
cc->InputSidePackets().Tag("FILE_SUFFIX").Get<std::string>(),
&filenames_));
return ::mediapipe::OkStatus();
}
::mediapipe::Status Process(CalculatorContext* cc) override {
if (current_output_ < filenames_.size()) {
auto contents = absl::make_unique<std::string>();
LOG(INFO) << filenames_[current_output_];
MP_RETURN_IF_ERROR(mediapipe::file::GetContents(
filenames_[current_output_], contents.get()));
++current_output_;
cc->Outputs()
.Tag("CONTENTS")
.Add(contents.release(), Timestamp(current_output_));
} else {
return tool::StatusStop();
}
return ::mediapipe::OkStatus();
}
private:
std::vector<std::string> filenames_;
int current_output_ = 0;
};
REGISTER_CALCULATOR(LocalFilePatternContentsCalculator);
} // namespace mediapipe
mediapipe/calculators/util/rect_to_render_data_calculator.cc
+9 -7
View File
@@ -45,15 +45,17 @@ RenderAnnotation::Rectangle* NewRect(
void SetRect(bool normalized, double xmin, double ymin, double width,
double height, double rotation,
RenderAnnotation::Rectangle* rect) {
if (xmin + width < 0.0 || ymin + height < 0.0) return;
if (normalized) {
if (xmin > 1.0 || ymin > 1.0) return;
if (rotation == 0.0) {
if (xmin + width < 0.0 || ymin + height < 0.0) return;
if (normalized) {
if (xmin > 1.0 || ymin > 1.0) return;
}
}
rect->set_normalized(normalized);
rect->set_left(normalized ? std::max(xmin, 0.0) : xmin);
rect->set_top(normalized ? std::max(ymin, 0.0) : ymin);
rect->set_right(normalized ? std::min(xmin + width, 1.0) : xmin + width);
rect->set_bottom(normalized ? std::min(ymin + height, 1.0) : ymin + height);
rect->set_left(xmin);
rect->set_top(ymin);
rect->set_right(xmin + width);
rect->set_bottom(ymin + height);
rect->set_rotation(rotation);
}
mediapipe/calculators/video/BUILD
+1
View File
@@ -368,6 +368,7 @@ cc_test(
cc_test(
name = "tvl1_optical_flow_calculator_test",
srcs = ["tvl1_optical_flow_calculator_test.cc"],
linkstatic = 1,
deps = [
":tvl1_optical_flow_calculator",
"//mediapipe/framework:calculator_framework",