Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3b6d3c4058 | ||
|
|
252a5713c7 |
@@ -15,6 +15,7 @@
|
||||
* [Hair Segmentation](mediapipe/docs/hair_segmentation_mobile_gpu.md) [[Web Demo]](https://viz.mediapipe.dev/runner/demos/hair_segmentation/hair_segmentation.html)
|
||||
* [Object Detection](mediapipe/docs/object_detection_mobile_gpu.md)
|
||||
* [Object Detection and Tracking](mediapipe/docs/object_tracking_mobile_gpu.md)
|
||||
* [Objectron: 3D Object Detection and Tracking](mediapipe/docs/objectron_mobile_gpu.md)
|
||||
* [AutoFlip](mediapipe/docs/autoflip.md)
|
||||
|
||||

|
||||
@@ -43,6 +44,8 @@ A web-based visualizer is hosted on [viz.mediapipe.dev](https://viz.mediapipe.de
|
||||
* [YouTube Channel](https://www.youtube.com/channel/UCObqmpuSMx-usADtL_qdMAw)
|
||||
|
||||
## Publications
|
||||
* [MediaPipe Objectron: Real-time 3D Object Detection on Mobile Devices](https://mediapipe.page.link/objectron-aiblog)
|
||||
* [AutoFlip: An Open Source Framework for Intelligent Video Reframing](https://mediapipe.page.link/autoflip)
|
||||
* [Google Developer Blog: MediaPipe on the Web](https://mediapipe.page.link/webdevblog)
|
||||
* [Google Developer Blog: Object Detection and Tracking using MediaPipe](https://mediapipe.page.link/objecttrackingblog)
|
||||
* [On-Device, Real-Time Hand Tracking with MediaPipe](https://ai.googleblog.com/2019/08/on-device-real-time-hand-tracking-with.html)
|
||||
@@ -63,7 +66,7 @@ A web-based visualizer is hosted on [viz.mediapipe.dev](https://viz.mediapipe.de
|
||||
* [Discuss](https://groups.google.com/forum/#!forum/mediapipe) - General community discussion around MediaPipe
|
||||
|
||||
## Alpha Disclaimer
|
||||
MediaPipe is currently in alpha for v0.6. We are still making breaking API changes and expect to get to stable API by v1.0.
|
||||
MediaPipe is currently in alpha for v0.7. We are still making breaking API changes and expect to get to stable API by v1.0.
|
||||
|
||||
## Contributing
|
||||
We welcome contributions. Please follow these [guidelines](./CONTRIBUTING.md).
|
||||
|
||||
@@ -10,15 +10,15 @@ http_archive(
|
||||
sha256 = "2ef429f5d7ce7111263289644d233707dba35e39696377ebab8b0bc701f7818e",
|
||||
)
|
||||
load("@bazel_skylib//lib:versions.bzl", "versions")
|
||||
versions.check(minimum_bazel_version = "0.24.1",
|
||||
versions.check(minimum_bazel_version = "1.0.0",
|
||||
maximum_bazel_version = "1.2.1")
|
||||
|
||||
|
||||
# ABSL cpp library lts_2019_08_08.
|
||||
# ABSL cpp library lts_2020_02_25
|
||||
http_archive(
|
||||
name = "com_google_absl",
|
||||
urls = [
|
||||
"https://github.com/abseil/abseil-cpp/archive/20190808.tar.gz",
|
||||
"https://github.com/abseil/abseil-cpp/archive/20200225.tar.gz",
|
||||
],
|
||||
# Remove after https://github.com/abseil/abseil-cpp/issues/326 is solved.
|
||||
patches = [
|
||||
@@ -27,8 +27,8 @@ http_archive(
|
||||
patch_args = [
|
||||
"-p1",
|
||||
],
|
||||
strip_prefix = "abseil-cpp-20190808",
|
||||
sha256 = "8100085dada279bf3ee00cd064d43b5f55e5d913be0dfe2906f06f8f28d5b37e"
|
||||
strip_prefix = "abseil-cpp-20200225",
|
||||
sha256 = "728a813291bdec2aa46eab8356ace9f75ac2ed9dfe2df5ab603c4e6c09f1c353"
|
||||
)
|
||||
|
||||
http_archive(
|
||||
@@ -117,18 +117,19 @@ http_archive(
|
||||
],
|
||||
)
|
||||
|
||||
# 2019-11-21
|
||||
_TENSORFLOW_GIT_COMMIT = "f482488b481a799ca07e7e2d153cf47b8e91a60c"
|
||||
_TENSORFLOW_SHA256= "8d9118c2ce186c7e1403f04b96982fe72c184060c7f7a93e30a28dca358694f0"
|
||||
# 2020-02-12
|
||||
# The last commit before TensorFlow switched to Bazel 2.0
|
||||
_TENSORFLOW_GIT_COMMIT = "77e9ffb9b2bfb1a4f7056e62d84039626923e328"
|
||||
_TENSORFLOW_SHA256= "176ccd82f7dd17c5e117b50d353603b129c7a6ccbfebd522ca47cc2a40f33f13"
|
||||
http_archive(
|
||||
name = "org_tensorflow",
|
||||
urls = [
|
||||
"https://mirror.bazel.build/github.com/tensorflow/tensorflow/archive/%s.tar.gz" % _TENSORFLOW_GIT_COMMIT,
|
||||
"https://github.com/tensorflow/tensorflow/archive/%s.tar.gz" % _TENSORFLOW_GIT_COMMIT,
|
||||
],
|
||||
# Patch https://github.com/tensorflow/tensorflow/commit/e3a7bdbebb99352351a19e2e403136166aa52934
|
||||
# A compatibility patch
|
||||
patches = [
|
||||
"@//third_party:org_tensorflow_e3a7bdbebb99352351a19e2e403136166aa52934.diff"
|
||||
"@//third_party:org_tensorflow_528e22eae8bf3206189a066032c66e9e5c9b4a61.diff"
|
||||
],
|
||||
patch_args = [
|
||||
"-p1",
|
||||
|
||||
@@ -610,6 +610,22 @@ cc_library(
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_test(
|
||||
name = "side_packet_to_stream_calculator_test",
|
||||
srcs = ["side_packet_to_stream_calculator_test.cc"],
|
||||
deps = [
|
||||
":side_packet_to_stream_calculator",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"//mediapipe/framework/port:integral_types",
|
||||
"//mediapipe/framework/port:parse_text_proto",
|
||||
"//mediapipe/framework/port:status",
|
||||
"//mediapipe/framework/tool:options_util",
|
||||
"@com_google_absl//absl/memory",
|
||||
"@com_google_absl//absl/strings",
|
||||
],
|
||||
)
|
||||
|
||||
cc_test(
|
||||
name = "immediate_mux_calculator_test",
|
||||
srcs = ["immediate_mux_calculator_test.cc"],
|
||||
|
||||
@@ -28,55 +28,133 @@ using mediapipe::PacketTypeSet;
|
||||
using mediapipe::Timestamp;
|
||||
|
||||
namespace {
|
||||
|
||||
constexpr char kTagAtPreStream[] = "AT_PRESTREAM";
|
||||
constexpr char kTagAtPostStream[] = "AT_POSTSTREAM";
|
||||
constexpr char kTagAtZero[] = "AT_ZERO";
|
||||
constexpr char kTagAtTick[] = "AT_TICK";
|
||||
constexpr char kTagTick[] = "TICK";
|
||||
|
||||
static std::map<std::string, Timestamp>* kTimestampMap = []() {
|
||||
auto* res = new std::map<std::string, Timestamp>();
|
||||
res->emplace("AT_PRESTREAM", Timestamp::PreStream());
|
||||
res->emplace("AT_POSTSTREAM", Timestamp::PostStream());
|
||||
res->emplace("AT_ZERO", Timestamp(0));
|
||||
res->emplace(kTagAtPreStream, Timestamp::PreStream());
|
||||
res->emplace(kTagAtPostStream, Timestamp::PostStream());
|
||||
res->emplace(kTagAtZero, Timestamp(0));
|
||||
res->emplace(kTagAtTick, Timestamp::Unset());
|
||||
return res;
|
||||
}();
|
||||
|
||||
template <typename CC>
|
||||
std::string GetOutputTag(const CC& cc) {
|
||||
// Single output tag only is required by contract.
|
||||
return *cc.Outputs().GetTags().begin();
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
// Outputs the single input_side_packet at the timestamp specified in the
|
||||
// output_stream tag. Valid tags are AT_PRESTREAM, AT_POSTSTREAM and AT_ZERO.
|
||||
// Outputs side packet(s) in corresponding output stream(s) with a particular
|
||||
// timestamp, depending on the tag used to define output stream(s). (One tag can
|
||||
// be used only.)
|
||||
//
|
||||
// Valid tags are AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO and AT_TICK and
|
||||
// corresponding timestamps are Timestamp::PreStream(), Timestamp::PostStream(),
|
||||
// Timestamp(0) and timestamp of a packet received in TICK input.
|
||||
//
|
||||
// Examples:
|
||||
// node {
|
||||
// calculator: "SidePacketToStreamCalculator"
|
||||
// input_side_packet: "side_packet"
|
||||
// output_stream: "AT_PRESTREAM:packet"
|
||||
// }
|
||||
//
|
||||
// node {
|
||||
// calculator: "SidePacketToStreamCalculator"
|
||||
// input_stream: "TICK:tick"
|
||||
// input_side_packet: "side_packet"
|
||||
// output_stream: "AT_TICK:packet"
|
||||
// }
|
||||
class SidePacketToStreamCalculator : public CalculatorBase {
|
||||
public:
|
||||
SidePacketToStreamCalculator() = default;
|
||||
~SidePacketToStreamCalculator() override = default;
|
||||
|
||||
static ::mediapipe::Status GetContract(CalculatorContract* cc);
|
||||
::mediapipe::Status Open(CalculatorContext* cc) override;
|
||||
::mediapipe::Status Process(CalculatorContext* cc) override;
|
||||
::mediapipe::Status Close(CalculatorContext* cc) override;
|
||||
|
||||
private:
|
||||
bool is_tick_processing_ = false;
|
||||
std::string output_tag_;
|
||||
};
|
||||
REGISTER_CALCULATOR(SidePacketToStreamCalculator);
|
||||
|
||||
::mediapipe::Status SidePacketToStreamCalculator::GetContract(
|
||||
CalculatorContract* cc) {
|
||||
cc->InputSidePackets().Index(0).SetAny();
|
||||
const auto& tags = cc->Outputs().GetTags();
|
||||
RET_CHECK(tags.size() == 1 && kTimestampMap->count(*tags.begin()) == 1)
|
||||
<< "Only one of AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO and AT_TICK tags is "
|
||||
"allowed and required to specify output stream(s).";
|
||||
RET_CHECK(
|
||||
(cc->Outputs().HasTag(kTagAtTick) && cc->Inputs().HasTag(kTagTick)) ||
|
||||
(!cc->Outputs().HasTag(kTagAtTick) && !cc->Inputs().HasTag(kTagTick)))
|
||||
<< "Either both of TICK and AT_TICK should be used or none of them.";
|
||||
const std::string output_tag = GetOutputTag(*cc);
|
||||
const int num_entries = cc->Outputs().NumEntries(output_tag);
|
||||
RET_CHECK_EQ(num_entries, cc->InputSidePackets().NumEntries())
|
||||
<< "Same number of input side packets and output streams is required.";
|
||||
for (int i = 0; i < num_entries; ++i) {
|
||||
cc->InputSidePackets().Index(i).SetAny();
|
||||
cc->Outputs()
|
||||
.Get(output_tag, i)
|
||||
.SetSameAs(cc->InputSidePackets().Index(i).GetSameAs());
|
||||
}
|
||||
|
||||
std::set<std::string> tags = cc->Outputs().GetTags();
|
||||
RET_CHECK_EQ(tags.size(), 1);
|
||||
if (cc->Inputs().HasTag(kTagTick)) {
|
||||
cc->Inputs().Tag(kTagTick).SetAny();
|
||||
}
|
||||
|
||||
RET_CHECK_EQ(kTimestampMap->count(*tags.begin()), 1);
|
||||
cc->Outputs().Tag(*tags.begin()).SetAny();
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status SidePacketToStreamCalculator::Open(CalculatorContext* cc) {
|
||||
output_tag_ = GetOutputTag(*cc);
|
||||
if (cc->Inputs().HasTag(kTagTick)) {
|
||||
is_tick_processing_ = true;
|
||||
// Set offset, so output timestamp bounds are updated in response to TICK
|
||||
// timestamp bound update.
|
||||
cc->SetOffset(TimestampDiff(0));
|
||||
}
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status SidePacketToStreamCalculator::Process(
|
||||
CalculatorContext* cc) {
|
||||
return mediapipe::tool::StatusStop();
|
||||
if (is_tick_processing_) {
|
||||
// TICK input is guaranteed to be non-empty, as it's the only input stream
|
||||
// for this calculator.
|
||||
const auto& timestamp = cc->Inputs().Tag(kTagTick).Value().Timestamp();
|
||||
for (int i = 0; i < cc->Outputs().NumEntries(output_tag_); ++i) {
|
||||
cc->Outputs()
|
||||
.Get(output_tag_, i)
|
||||
.AddPacket(cc->InputSidePackets().Index(i).At(timestamp));
|
||||
}
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
return ::mediapipe::tool::StatusStop();
|
||||
}
|
||||
|
||||
::mediapipe::Status SidePacketToStreamCalculator::Close(CalculatorContext* cc) {
|
||||
std::set<std::string> tags = cc->Outputs().GetTags();
|
||||
RET_CHECK_EQ(tags.size(), 1);
|
||||
const std::string& tag = *tags.begin();
|
||||
RET_CHECK_EQ(kTimestampMap->count(tag), 1);
|
||||
cc->Outputs().Tag(tag).AddPacket(
|
||||
cc->InputSidePackets().Index(0).At(kTimestampMap->at(tag)));
|
||||
|
||||
if (!cc->Outputs().HasTag(kTagAtTick)) {
|
||||
const auto& timestamp = kTimestampMap->at(output_tag_);
|
||||
for (int i = 0; i < cc->Outputs().NumEntries(output_tag_); ++i) {
|
||||
cc->Outputs()
|
||||
.Get(output_tag_, i)
|
||||
.AddPacket(cc->InputSidePackets().Index(i).At(timestamp));
|
||||
}
|
||||
}
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,275 @@
|
||||
// Copyright 2020 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "absl/memory/memory.h"
|
||||
#include "absl/strings/match.h"
|
||||
#include "absl/strings/str_replace.h"
|
||||
#include "absl/strings/string_view.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/port/gtest.h"
|
||||
#include "mediapipe/framework/port/integral_types.h"
|
||||
#include "mediapipe/framework/port/parse_text_proto.h"
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
#include "mediapipe/framework/port/status_matchers.h"
|
||||
#include "mediapipe/framework/tool/options_util.h"
|
||||
|
||||
namespace mediapipe {
|
||||
namespace {
|
||||
|
||||
TEST(SidePacketToStreamCalculator, WrongConfig_MissingTick) {
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
input_stream: "tick"
|
||||
input_side_packet: "side_packet"
|
||||
output_stream: "packet"
|
||||
node {
|
||||
calculator: "SidePacketToStreamCalculator"
|
||||
input_side_packet: "side_packet"
|
||||
output_stream: "AT_TICK:packet"
|
||||
}
|
||||
)");
|
||||
CalculatorGraph graph;
|
||||
auto status = graph.Initialize(graph_config);
|
||||
EXPECT_FALSE(status.ok());
|
||||
EXPECT_PRED2(
|
||||
absl::StrContains, status.message(),
|
||||
"Either both of TICK and AT_TICK should be used or none of them.");
|
||||
}
|
||||
|
||||
TEST(SidePacketToStreamCalculator, WrongConfig_NonExistentTag) {
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
input_stream: "tick"
|
||||
input_side_packet: "side_packet"
|
||||
output_stream: "packet"
|
||||
node {
|
||||
calculator: "SidePacketToStreamCalculator"
|
||||
input_side_packet: "side_packet"
|
||||
output_stream: "DOES_NOT_EXIST:packet"
|
||||
}
|
||||
)");
|
||||
CalculatorGraph graph;
|
||||
auto status = graph.Initialize(graph_config);
|
||||
EXPECT_FALSE(status.ok());
|
||||
EXPECT_PRED2(absl::StrContains, status.message(),
|
||||
"Only one of AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO and AT_TICK "
|
||||
"tags is allowed and required to specify output stream(s).");
|
||||
}
|
||||
|
||||
TEST(SidePacketToStreamCalculator, WrongConfig_MixedTags) {
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
input_stream: "tick"
|
||||
input_side_packet: "side_packet0"
|
||||
input_side_packet: "side_packet1"
|
||||
node {
|
||||
calculator: "SidePacketToStreamCalculator"
|
||||
input_side_packet: "side_packet0"
|
||||
input_side_packet: "side_packet1"
|
||||
output_stream: "AT_TICK:packet0"
|
||||
output_stream: "AT_PRE_STREAM:packet1"
|
||||
}
|
||||
)");
|
||||
CalculatorGraph graph;
|
||||
auto status = graph.Initialize(graph_config);
|
||||
EXPECT_FALSE(status.ok());
|
||||
EXPECT_PRED2(absl::StrContains, status.message(),
|
||||
"Only one of AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO and AT_TICK "
|
||||
"tags is allowed and required to specify output stream(s).");
|
||||
}
|
||||
|
||||
TEST(SidePacketToStreamCalculator, WrongConfig_NotEnoughSidePackets) {
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
input_side_packet: "side_packet0"
|
||||
input_side_packet: "side_packet1"
|
||||
node {
|
||||
calculator: "SidePacketToStreamCalculator"
|
||||
input_side_packet: "side_packet0"
|
||||
output_stream: "AT_PRESTREAM:0:packet0"
|
||||
output_stream: "AT_PRESTREAM:1:packet1"
|
||||
}
|
||||
)");
|
||||
CalculatorGraph graph;
|
||||
auto status = graph.Initialize(graph_config);
|
||||
EXPECT_FALSE(status.ok());
|
||||
EXPECT_PRED2(
|
||||
absl::StrContains, status.message(),
|
||||
"Same number of input side packets and output streams is required.");
|
||||
}
|
||||
|
||||
TEST(SidePacketToStreamCalculator, WrongConfig_NotEnoughOutputStreams) {
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
input_side_packet: "side_packet0"
|
||||
input_side_packet: "side_packet1"
|
||||
node {
|
||||
calculator: "SidePacketToStreamCalculator"
|
||||
input_side_packet: "side_packet0"
|
||||
input_side_packet: "side_packet1"
|
||||
output_stream: "AT_PRESTREAM:packet0"
|
||||
}
|
||||
)");
|
||||
CalculatorGraph graph;
|
||||
auto status = graph.Initialize(graph_config);
|
||||
EXPECT_FALSE(status.ok());
|
||||
EXPECT_PRED2(
|
||||
absl::StrContains, status.message(),
|
||||
"Same number of input side packets and output streams is required.");
|
||||
}
|
||||
|
||||
void DoTestNonAtTickOutputTag(absl::string_view tag,
|
||||
Timestamp expected_timestamp) {
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(absl::StrReplaceAll(
|
||||
R"(
|
||||
input_side_packet: "side_packet"
|
||||
output_stream: "packet"
|
||||
node {
|
||||
calculator: "SidePacketToStreamCalculator"
|
||||
input_side_packet: "side_packet"
|
||||
output_stream: "$tag:packet"
|
||||
}
|
||||
)",
|
||||
{{"$tag", tag}}));
|
||||
CalculatorGraph graph;
|
||||
MP_ASSERT_OK(graph.Initialize(graph_config));
|
||||
const int expected_value = 10;
|
||||
std::vector<Packet> output_packets;
|
||||
MP_ASSERT_OK(graph.ObserveOutputStream(
|
||||
"packet", [&output_packets](const Packet& packet) {
|
||||
output_packets.push_back(packet);
|
||||
return ::mediapipe::OkStatus();
|
||||
}));
|
||||
MP_ASSERT_OK(
|
||||
graph.StartRun({{"side_packet", MakePacket<int>(expected_value)}}));
|
||||
MP_ASSERT_OK(graph.WaitForObservedOutput());
|
||||
|
||||
ASSERT_FALSE(output_packets.empty());
|
||||
EXPECT_EQ(expected_timestamp, output_packets.back().Timestamp());
|
||||
EXPECT_EQ(expected_value, output_packets.back().Get<int>());
|
||||
}
|
||||
|
||||
TEST(SidePacketToStreamCalculator, NoAtTickOutputTags) {
|
||||
DoTestNonAtTickOutputTag("AT_PRESTREAM", Timestamp::PreStream());
|
||||
DoTestNonAtTickOutputTag("AT_POSTSTREAM", Timestamp::PostStream());
|
||||
DoTestNonAtTickOutputTag("AT_ZERO", Timestamp(0));
|
||||
}
|
||||
|
||||
TEST(SidePacketToStreamCalculator, AtTick) {
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
input_stream: "tick"
|
||||
input_side_packet: "side_packet"
|
||||
output_stream: "packet"
|
||||
node {
|
||||
calculator: "SidePacketToStreamCalculator"
|
||||
input_stream: "TICK:tick"
|
||||
input_side_packet: "side_packet"
|
||||
output_stream: "AT_TICK:packet"
|
||||
}
|
||||
)");
|
||||
std::vector<Packet> output_packets;
|
||||
tool::AddVectorSink("packet", &graph_config, &output_packets);
|
||||
CalculatorGraph graph;
|
||||
|
||||
MP_ASSERT_OK(graph.Initialize(graph_config));
|
||||
const int expected_value = 20;
|
||||
MP_ASSERT_OK(
|
||||
graph.StartRun({{"side_packet", MakePacket<int>(expected_value)}}));
|
||||
|
||||
auto tick_and_verify = [&graph, &output_packets,
|
||||
expected_value](int at_timestamp) {
|
||||
MP_ASSERT_OK(graph.AddPacketToInputStream(
|
||||
"tick",
|
||||
MakePacket<int>(/*doesn't matter*/ 1).At(Timestamp(at_timestamp))));
|
||||
MP_ASSERT_OK(graph.WaitUntilIdle());
|
||||
|
||||
ASSERT_FALSE(output_packets.empty());
|
||||
EXPECT_EQ(Timestamp(at_timestamp), output_packets.back().Timestamp());
|
||||
EXPECT_EQ(expected_value, output_packets.back().Get<int>());
|
||||
};
|
||||
|
||||
tick_and_verify(/*at_timestamp=*/0);
|
||||
tick_and_verify(/*at_timestamp=*/1);
|
||||
tick_and_verify(/*at_timestamp=*/128);
|
||||
tick_and_verify(/*at_timestamp=*/1024);
|
||||
tick_and_verify(/*at_timestamp=*/1025);
|
||||
}
|
||||
|
||||
TEST(SidePacketToStreamCalculator, AtTick_MultipleSidePackets) {
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
input_stream: "tick"
|
||||
input_side_packet: "side_packet0"
|
||||
input_side_packet: "side_packet1"
|
||||
output_stream: "packet0"
|
||||
output_stream: "packet1"
|
||||
node {
|
||||
calculator: "SidePacketToStreamCalculator"
|
||||
input_stream: "TICK:tick"
|
||||
input_side_packet: "side_packet0"
|
||||
input_side_packet: "side_packet1"
|
||||
output_stream: "AT_TICK:0:packet0"
|
||||
output_stream: "AT_TICK:1:packet1"
|
||||
}
|
||||
)");
|
||||
std::vector<Packet> output_packets0;
|
||||
tool::AddVectorSink("packet0", &graph_config, &output_packets0);
|
||||
std::vector<Packet> output_packets1;
|
||||
tool::AddVectorSink("packet1", &graph_config, &output_packets1);
|
||||
CalculatorGraph graph;
|
||||
|
||||
MP_ASSERT_OK(graph.Initialize(graph_config));
|
||||
const int expected_value0 = 20;
|
||||
const int expected_value1 = 128;
|
||||
MP_ASSERT_OK(
|
||||
graph.StartRun({{"side_packet0", MakePacket<int>(expected_value0)},
|
||||
{"side_packet1", MakePacket<int>(expected_value1)}}));
|
||||
|
||||
auto tick_and_verify = [&graph, &output_packets0, &output_packets1,
|
||||
expected_value0, expected_value1](int at_timestamp) {
|
||||
MP_ASSERT_OK(graph.AddPacketToInputStream(
|
||||
"tick",
|
||||
MakePacket<int>(/*doesn't matter*/ 1).At(Timestamp(at_timestamp))));
|
||||
MP_ASSERT_OK(graph.WaitUntilIdle());
|
||||
|
||||
ASSERT_FALSE(output_packets0.empty());
|
||||
ASSERT_FALSE(output_packets1.empty());
|
||||
|
||||
EXPECT_EQ(Timestamp(at_timestamp), output_packets0.back().Timestamp());
|
||||
EXPECT_EQ(expected_value0, output_packets0.back().Get<int>());
|
||||
EXPECT_EQ(Timestamp(at_timestamp), output_packets1.back().Timestamp());
|
||||
EXPECT_EQ(expected_value1, output_packets1.back().Get<int>());
|
||||
};
|
||||
|
||||
tick_and_verify(/*at_timestamp=*/0);
|
||||
tick_and_verify(/*at_timestamp=*/1);
|
||||
tick_and_verify(/*at_timestamp=*/128);
|
||||
tick_and_verify(/*at_timestamp=*/1024);
|
||||
tick_and_verify(/*at_timestamp=*/1025);
|
||||
}
|
||||
|
||||
} // namespace
|
||||
} // namespace mediapipe
|
||||
@@ -346,9 +346,7 @@ cc_library(
|
||||
],
|
||||
"//conditions:default": [],
|
||||
}),
|
||||
visibility = [
|
||||
"//visibility:public",
|
||||
],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
":image_cropping_calculator_cc_proto",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
|
||||
@@ -75,6 +75,11 @@ class ColorConvertCalculator : public CalculatorBase {
|
||||
static ::mediapipe::Status GetContract(CalculatorContract* cc);
|
||||
::mediapipe::Status Process(CalculatorContext* cc) override;
|
||||
|
||||
::mediapipe::Status Open(CalculatorContext* cc) override {
|
||||
cc->SetOffset(TimestampDiff(0));
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
private:
|
||||
// Wrangles the appropriate inputs and outputs to perform the color
|
||||
// conversion. The ImageFrame on input_tag is converted using the
|
||||
|
||||
@@ -75,11 +75,28 @@ REGISTER_CALCULATOR(ImageCroppingCalculator);
|
||||
}
|
||||
#endif // !MEDIAPIPE_DISABLE_GPU
|
||||
|
||||
RET_CHECK(cc->Inputs().HasTag(kRectTag) ^ cc->Inputs().HasTag(kNormRectTag) ^
|
||||
(cc->Options<mediapipe::ImageCroppingCalculatorOptions>()
|
||||
.has_norm_width() &&
|
||||
cc->Options<mediapipe::ImageCroppingCalculatorOptions>()
|
||||
.has_norm_height()));
|
||||
int flags = 0;
|
||||
if (cc->Inputs().HasTag(kRectTag)) {
|
||||
++flags;
|
||||
}
|
||||
if (cc->Inputs().HasTag(kWidthTag) && cc->Inputs().HasTag(kHeightTag)) {
|
||||
++flags;
|
||||
}
|
||||
if (cc->Inputs().HasTag(kNormRectTag)) {
|
||||
++flags;
|
||||
}
|
||||
if (cc->Options<mediapipe::ImageCroppingCalculatorOptions>()
|
||||
.has_norm_width() &&
|
||||
cc->Options<mediapipe::ImageCroppingCalculatorOptions>()
|
||||
.has_norm_height()) {
|
||||
++flags;
|
||||
}
|
||||
if (cc->Options<mediapipe::ImageCroppingCalculatorOptions>().has_width() &&
|
||||
cc->Options<mediapipe::ImageCroppingCalculatorOptions>().has_height()) {
|
||||
++flags;
|
||||
}
|
||||
RET_CHECK(flags == 1) << "Illegal combination of input streams/options.";
|
||||
|
||||
if (cc->Inputs().HasTag(kRectTag)) {
|
||||
cc->Inputs().Tag(kRectTag).Set<Rect>();
|
||||
}
|
||||
@@ -119,6 +136,13 @@ REGISTER_CALCULATOR(ImageCroppingCalculator);
|
||||
#endif // !MEDIAPIPE_DISABLE_GPU
|
||||
}
|
||||
|
||||
// Validate border mode.
|
||||
if (use_gpu_) {
|
||||
MP_RETURN_IF_ERROR(ValidateBorderModeForGPU(cc));
|
||||
} else {
|
||||
MP_RETURN_IF_ERROR(ValidateBorderModeForCPU(cc));
|
||||
}
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
@@ -162,6 +186,32 @@ REGISTER_CALCULATOR(ImageCroppingCalculator);
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status ImageCroppingCalculator::ValidateBorderModeForCPU(
|
||||
CalculatorContext* cc) {
|
||||
int border_mode;
|
||||
return GetBorderModeForOpenCV(cc, &border_mode);
|
||||
}
|
||||
|
||||
::mediapipe::Status ImageCroppingCalculator::ValidateBorderModeForGPU(
|
||||
CalculatorContext* cc) {
|
||||
mediapipe::ImageCroppingCalculatorOptions options =
|
||||
cc->Options<mediapipe::ImageCroppingCalculatorOptions>();
|
||||
|
||||
switch (options.border_mode()) {
|
||||
case mediapipe::ImageCroppingCalculatorOptions::BORDER_ZERO:
|
||||
LOG(WARNING) << "BORDER_ZERO mode is not supported by GPU "
|
||||
<< "implementation and will fall back into BORDER_REPLICATE";
|
||||
break;
|
||||
case mediapipe::ImageCroppingCalculatorOptions::BORDER_REPLICATE:
|
||||
break;
|
||||
default:
|
||||
RET_CHECK_FAIL() << "Unsupported border mode for GPU: "
|
||||
<< options.border_mode();
|
||||
}
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status ImageCroppingCalculator::RenderCpu(CalculatorContext* cc) {
|
||||
if (cc->Inputs().Tag(kImageTag).IsEmpty()) {
|
||||
return ::mediapipe::OkStatus();
|
||||
@@ -172,6 +222,10 @@ REGISTER_CALCULATOR(ImageCroppingCalculator);
|
||||
auto [target_width, target_height, rect_center_x, rect_center_y, rotation] =
|
||||
GetCropSpecs(cc, input_img.Width(), input_img.Height());
|
||||
|
||||
// Get border mode and value for OpenCV.
|
||||
int border_mode;
|
||||
MP_RETURN_IF_ERROR(GetBorderModeForOpenCV(cc, &border_mode));
|
||||
|
||||
const cv::RotatedRect min_rect(cv::Point2f(rect_center_x, rect_center_y),
|
||||
cv::Size2f(target_width, target_height),
|
||||
rotation * 180.f / M_PI);
|
||||
@@ -191,7 +245,9 @@ REGISTER_CALCULATOR(ImageCroppingCalculator);
|
||||
cv::getPerspectiveTransform(src_points, dst_points);
|
||||
cv::Mat cropped_image;
|
||||
cv::warpPerspective(input_mat, cropped_image, projection_matrix,
|
||||
cv::Size(min_rect.size.width, min_rect.size.height));
|
||||
cv::Size(min_rect.size.width, min_rect.size.height),
|
||||
/* flags = */ 0,
|
||||
/* borderMode = */ border_mode);
|
||||
|
||||
std::unique_ptr<ImageFrame> output_frame(new ImageFrame(
|
||||
input_img.Format(), cropped_image.cols, cropped_image.rows));
|
||||
@@ -453,6 +509,7 @@ RectSpec ImageCroppingCalculator::GetCropSpecs(const CalculatorContext* cc,
|
||||
rotation = options.rotation();
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
.width = crop_width,
|
||||
.height = crop_height,
|
||||
@@ -462,4 +519,24 @@ RectSpec ImageCroppingCalculator::GetCropSpecs(const CalculatorContext* cc,
|
||||
};
|
||||
}
|
||||
|
||||
::mediapipe::Status ImageCroppingCalculator::GetBorderModeForOpenCV(
|
||||
CalculatorContext* cc, int* border_mode) {
|
||||
mediapipe::ImageCroppingCalculatorOptions options =
|
||||
cc->Options<mediapipe::ImageCroppingCalculatorOptions>();
|
||||
|
||||
switch (options.border_mode()) {
|
||||
case mediapipe::ImageCroppingCalculatorOptions::BORDER_ZERO:
|
||||
*border_mode = cv::BORDER_CONSTANT;
|
||||
break;
|
||||
case mediapipe::ImageCroppingCalculatorOptions::BORDER_REPLICATE:
|
||||
*border_mode = cv::BORDER_REPLICATE;
|
||||
break;
|
||||
default:
|
||||
RET_CHECK_FAIL() << "Unsupported border mode for CPU: "
|
||||
<< options.border_mode();
|
||||
}
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
} // namespace mediapipe
|
||||
|
||||
@@ -36,6 +36,7 @@
|
||||
// Note: input_stream values take precedence over options defined in the graph.
|
||||
//
|
||||
namespace mediapipe {
|
||||
|
||||
struct RectSpec {
|
||||
int width;
|
||||
int height;
|
||||
@@ -63,12 +64,16 @@ class ImageCroppingCalculator : public CalculatorBase {
|
||||
int src_height);
|
||||
|
||||
private:
|
||||
::mediapipe::Status ValidateBorderModeForCPU(CalculatorContext* cc);
|
||||
::mediapipe::Status ValidateBorderModeForGPU(CalculatorContext* cc);
|
||||
::mediapipe::Status RenderCpu(CalculatorContext* cc);
|
||||
::mediapipe::Status RenderGpu(CalculatorContext* cc);
|
||||
::mediapipe::Status InitGpu(CalculatorContext* cc);
|
||||
void GlRender();
|
||||
void GetOutputDimensions(CalculatorContext* cc, int src_width, int src_height,
|
||||
int* dst_width, int* dst_height);
|
||||
::mediapipe::Status GetBorderModeForOpenCV(CalculatorContext* cc,
|
||||
int* border_mode);
|
||||
|
||||
mediapipe::ImageCroppingCalculatorOptions options_;
|
||||
|
||||
|
||||
@@ -40,4 +40,15 @@ message ImageCroppingCalculatorOptions {
|
||||
// The (0, 0) point is at the (top, left) corner.
|
||||
optional float norm_center_x = 6 [default = 0];
|
||||
optional float norm_center_y = 7 [default = 0];
|
||||
|
||||
enum BorderMode {
|
||||
// First unspecified value is required by the guideline. See details here:
|
||||
// https://developers.google.com/protocol-buffers/docs/style#enums
|
||||
BORDER_UNSPECIFIED = 0;
|
||||
BORDER_ZERO = 1;
|
||||
BORDER_REPLICATE = 2;
|
||||
}
|
||||
|
||||
// Specifies behaviour for crops that go beyond image borders.
|
||||
optional BorderMode border_mode = 8 [default = BORDER_ZERO];
|
||||
}
|
||||
|
||||
@@ -56,11 +56,11 @@ TEST(ImageCroppingCalculatorTest, GetCroppingDimensionsNormal) {
|
||||
}
|
||||
)");
|
||||
|
||||
auto calculator_state =
|
||||
CalculatorState("Node", 0, "Calculator", calculator_node, nullptr);
|
||||
auto cc =
|
||||
CalculatorContext(&calculator_state, tool::CreateTagMap({}).ValueOrDie(),
|
||||
tool::CreateTagMap({}).ValueOrDie());
|
||||
auto calculator_state = absl::make_unique<CalculatorState>(
|
||||
"Node", 0, "Calculator", calculator_node, nullptr);
|
||||
auto cc = absl::make_unique<CalculatorContext>(
|
||||
calculator_state.get(), tool::CreateTagMap({}).ValueOrDie(),
|
||||
tool::CreateTagMap({}).ValueOrDie());
|
||||
|
||||
RectSpec expectRect = {
|
||||
.width = 60,
|
||||
@@ -69,9 +69,9 @@ TEST(ImageCroppingCalculatorTest, GetCroppingDimensionsNormal) {
|
||||
.center_y = 50,
|
||||
.rotation = 0.3,
|
||||
};
|
||||
EXPECT_EQ(
|
||||
ImageCroppingCalculator::GetCropSpecs(&cc, input_width, input_height),
|
||||
expectRect);
|
||||
EXPECT_EQ(ImageCroppingCalculator::GetCropSpecs(cc.get(), input_width,
|
||||
input_height),
|
||||
expectRect);
|
||||
} // TEST
|
||||
|
||||
// Test when (width height) + (norm_width norm_height) are set in options.
|
||||
@@ -96,11 +96,11 @@ TEST(ImageCroppingCalculatorTest, RedundantSpecInOptions) {
|
||||
}
|
||||
)");
|
||||
|
||||
auto calculator_state =
|
||||
CalculatorState("Node", 0, "Calculator", calculator_node, nullptr);
|
||||
auto cc =
|
||||
CalculatorContext(&calculator_state, tool::CreateTagMap({}).ValueOrDie(),
|
||||
tool::CreateTagMap({}).ValueOrDie());
|
||||
auto calculator_state = absl::make_unique<CalculatorState>(
|
||||
"Node", 0, "Calculator", calculator_node, nullptr);
|
||||
auto cc = absl::make_unique<CalculatorContext>(
|
||||
calculator_state.get(), tool::CreateTagMap({}).ValueOrDie(),
|
||||
tool::CreateTagMap({}).ValueOrDie());
|
||||
RectSpec expectRect = {
|
||||
.width = 50,
|
||||
.height = 50,
|
||||
@@ -108,9 +108,9 @@ TEST(ImageCroppingCalculatorTest, RedundantSpecInOptions) {
|
||||
.center_y = 50,
|
||||
.rotation = 0.3,
|
||||
};
|
||||
EXPECT_EQ(
|
||||
ImageCroppingCalculator::GetCropSpecs(&cc, input_width, input_height),
|
||||
expectRect);
|
||||
EXPECT_EQ(ImageCroppingCalculator::GetCropSpecs(cc.get(), input_width,
|
||||
input_height),
|
||||
expectRect);
|
||||
} // TEST
|
||||
|
||||
// Test when WIDTH HEIGHT are set from input stream,
|
||||
@@ -138,16 +138,16 @@ TEST(ImageCroppingCalculatorTest, RedundantSpectWithInputStream) {
|
||||
}
|
||||
)");
|
||||
|
||||
auto calculator_state =
|
||||
CalculatorState("Node", 0, "Calculator", calculator_node, nullptr);
|
||||
auto calculator_state = absl::make_unique<CalculatorState>(
|
||||
"Node", 0, "Calculator", calculator_node, nullptr);
|
||||
auto inputTags = tool::CreateTagMap({
|
||||
"HEIGHT:0:crop_height",
|
||||
"WIDTH:0:crop_width",
|
||||
})
|
||||
.ValueOrDie();
|
||||
auto cc = CalculatorContext(&calculator_state, inputTags,
|
||||
tool::CreateTagMap({}).ValueOrDie());
|
||||
auto& inputs = cc.Inputs();
|
||||
auto cc = absl::make_unique<CalculatorContext>(
|
||||
calculator_state.get(), inputTags, tool::CreateTagMap({}).ValueOrDie());
|
||||
auto& inputs = cc->Inputs();
|
||||
inputs.Tag(kHeightTag).Value() = MakePacket<int>(1);
|
||||
inputs.Tag(kWidthTag).Value() = MakePacket<int>(1);
|
||||
RectSpec expectRect = {
|
||||
@@ -157,9 +157,9 @@ TEST(ImageCroppingCalculatorTest, RedundantSpectWithInputStream) {
|
||||
.center_y = 50,
|
||||
.rotation = 0.3,
|
||||
};
|
||||
EXPECT_EQ(
|
||||
ImageCroppingCalculator::GetCropSpecs(&cc, input_width, input_height),
|
||||
expectRect);
|
||||
EXPECT_EQ(ImageCroppingCalculator::GetCropSpecs(cc.get(), input_width,
|
||||
input_height),
|
||||
expectRect);
|
||||
} // TEST
|
||||
|
||||
// Test when RECT is set from input stream,
|
||||
@@ -186,15 +186,15 @@ TEST(ImageCroppingCalculatorTest, RedundantSpecWithInputStream) {
|
||||
}
|
||||
)");
|
||||
|
||||
auto calculator_state =
|
||||
CalculatorState("Node", 0, "Calculator", calculator_node, nullptr);
|
||||
auto calculator_state = absl::make_unique<CalculatorState>(
|
||||
"Node", 0, "Calculator", calculator_node, nullptr);
|
||||
auto inputTags = tool::CreateTagMap({
|
||||
"RECT:0:rect",
|
||||
})
|
||||
.ValueOrDie();
|
||||
auto cc = CalculatorContext(&calculator_state, inputTags,
|
||||
tool::CreateTagMap({}).ValueOrDie());
|
||||
auto& inputs = cc.Inputs();
|
||||
auto cc = absl::make_unique<CalculatorContext>(
|
||||
calculator_state.get(), inputTags, tool::CreateTagMap({}).ValueOrDie());
|
||||
auto& inputs = cc->Inputs();
|
||||
mediapipe::Rect rect = ParseTextProtoOrDie<mediapipe::Rect>(
|
||||
R"(
|
||||
width: 1 height: 1 x_center: 40 y_center: 40 rotation: 0.5
|
||||
@@ -207,9 +207,9 @@ TEST(ImageCroppingCalculatorTest, RedundantSpecWithInputStream) {
|
||||
.center_y = 40,
|
||||
.rotation = 0.5,
|
||||
};
|
||||
EXPECT_EQ(
|
||||
ImageCroppingCalculator::GetCropSpecs(&cc, input_width, input_height),
|
||||
expectRect);
|
||||
EXPECT_EQ(ImageCroppingCalculator::GetCropSpecs(cc.get(), input_width,
|
||||
input_height),
|
||||
expectRect);
|
||||
} // TEST
|
||||
|
||||
} // namespace
|
||||
|
||||
@@ -104,6 +104,14 @@ mediapipe::ScaleMode_Mode ParseScaleMode(
|
||||
// to be a multiple of 90 degrees. If provided, it overrides the
|
||||
// ROTATION_DEGREES input side packet.
|
||||
//
|
||||
// FLIP_HORIZONTALLY (optional): Whether to flip image horizontally or not. If
|
||||
// provided, it overrides the FLIP_HORIZONTALLY input side packet and/or
|
||||
// corresponding field in the calculator options.
|
||||
//
|
||||
// FLIP_VERTICALLY (optional): Whether to flip image vertically or not. If
|
||||
// provided, it overrides the FLIP_VERTICALLY input side packet and/or
|
||||
// corresponding field in the calculator options.
|
||||
//
|
||||
// Output:
|
||||
// One of the following two tags:
|
||||
// IMAGE - ImageFrame representing the output image.
|
||||
@@ -129,6 +137,12 @@ mediapipe::ScaleMode_Mode ParseScaleMode(
|
||||
// degrees. It has to be a multiple of 90 degrees. It overrides the
|
||||
// corresponding field in the calculator options.
|
||||
//
|
||||
// FLIP_HORIZONTALLY (optional): Whether to flip image horizontally or not.
|
||||
// It overrides the corresponding field in the calculator options.
|
||||
//
|
||||
// FLIP_VERTICALLY (optional): Whether to flip image vertically or not.
|
||||
// It overrides the corresponding field in the calculator options.
|
||||
//
|
||||
// Calculator options (see image_transformation_calculator.proto):
|
||||
// output_width, output_height - (optional) Desired scaled image size.
|
||||
// rotation_mode - (optional) Rotation in multiples of 90 degrees.
|
||||
@@ -167,6 +181,8 @@ class ImageTransformationCalculator : public CalculatorBase {
|
||||
int output_height_ = 0;
|
||||
mediapipe::RotationMode_Mode rotation_;
|
||||
mediapipe::ScaleMode_Mode scale_mode_;
|
||||
bool flip_horizontally_ = false;
|
||||
bool flip_vertically_ = false;
|
||||
|
||||
bool use_gpu_ = false;
|
||||
#if !defined(MEDIAPIPE_DISABLE_GPU)
|
||||
@@ -203,6 +219,12 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
|
||||
if (cc->Inputs().HasTag("ROTATION_DEGREES")) {
|
||||
cc->Inputs().Tag("ROTATION_DEGREES").Set<int>();
|
||||
}
|
||||
if (cc->Inputs().HasTag("FLIP_HORIZONTALLY")) {
|
||||
cc->Inputs().Tag("FLIP_HORIZONTALLY").Set<bool>();
|
||||
}
|
||||
if (cc->Inputs().HasTag("FLIP_VERTICALLY")) {
|
||||
cc->Inputs().Tag("FLIP_VERTICALLY").Set<bool>();
|
||||
}
|
||||
|
||||
if (cc->InputSidePackets().HasTag("OUTPUT_DIMENSIONS")) {
|
||||
cc->InputSidePackets().Tag("OUTPUT_DIMENSIONS").Set<DimensionsPacketType>();
|
||||
@@ -210,6 +232,12 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
|
||||
if (cc->InputSidePackets().HasTag("ROTATION_DEGREES")) {
|
||||
cc->InputSidePackets().Tag("ROTATION_DEGREES").Set<int>();
|
||||
}
|
||||
if (cc->InputSidePackets().HasTag("FLIP_HORIZONTALLY")) {
|
||||
cc->InputSidePackets().Tag("FLIP_HORIZONTALLY").Set<bool>();
|
||||
}
|
||||
if (cc->InputSidePackets().HasTag("FLIP_VERTICALLY")) {
|
||||
cc->InputSidePackets().Tag("FLIP_VERTICALLY").Set<bool>();
|
||||
}
|
||||
|
||||
if (cc->Outputs().HasTag("LETTERBOX_PADDING")) {
|
||||
cc->Outputs().Tag("LETTERBOX_PADDING").Set<std::array<float, 4>>();
|
||||
@@ -245,6 +273,7 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
|
||||
output_width_ = options_.output_width();
|
||||
output_height_ = options_.output_height();
|
||||
}
|
||||
|
||||
if (cc->InputSidePackets().HasTag("ROTATION_DEGREES")) {
|
||||
rotation_ = DegreesToRotationMode(
|
||||
cc->InputSidePackets().Tag("ROTATION_DEGREES").Get<int>());
|
||||
@@ -252,6 +281,20 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
|
||||
rotation_ = options_.rotation_mode();
|
||||
}
|
||||
|
||||
if (cc->InputSidePackets().HasTag("FLIP_HORIZONTALLY")) {
|
||||
flip_horizontally_ =
|
||||
cc->InputSidePackets().Tag("FLIP_HORIZONTALLY").Get<bool>();
|
||||
} else {
|
||||
flip_horizontally_ = options_.flip_horizontally();
|
||||
}
|
||||
|
||||
if (cc->InputSidePackets().HasTag("FLIP_VERTICALLY")) {
|
||||
flip_vertically_ =
|
||||
cc->InputSidePackets().Tag("FLIP_VERTICALLY").Get<bool>();
|
||||
} else {
|
||||
flip_vertically_ = options_.flip_vertically();
|
||||
}
|
||||
|
||||
scale_mode_ = ParseScaleMode(options_.scale_mode(), DEFAULT_SCALE_MODE);
|
||||
|
||||
if (use_gpu_) {
|
||||
@@ -268,12 +311,37 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
|
||||
|
||||
::mediapipe::Status ImageTransformationCalculator::Process(
|
||||
CalculatorContext* cc) {
|
||||
// Override values if specified so.
|
||||
if (cc->Inputs().HasTag("ROTATION_DEGREES") &&
|
||||
!cc->Inputs().Tag("ROTATION_DEGREES").IsEmpty()) {
|
||||
rotation_ =
|
||||
DegreesToRotationMode(cc->Inputs().Tag("ROTATION_DEGREES").Get<int>());
|
||||
}
|
||||
if (cc->Inputs().HasTag("FLIP_HORIZONTALLY") &&
|
||||
!cc->Inputs().Tag("FLIP_HORIZONTALLY").IsEmpty()) {
|
||||
flip_horizontally_ = cc->Inputs().Tag("FLIP_HORIZONTALLY").Get<bool>();
|
||||
}
|
||||
if (cc->Inputs().HasTag("FLIP_VERTICALLY") &&
|
||||
!cc->Inputs().Tag("FLIP_VERTICALLY").IsEmpty()) {
|
||||
flip_vertically_ = cc->Inputs().Tag("FLIP_VERTICALLY").Get<bool>();
|
||||
}
|
||||
|
||||
if (use_gpu_) {
|
||||
#if !defined(MEDIAPIPE_DISABLE_GPU)
|
||||
if (cc->Inputs().Tag("IMAGE_GPU").IsEmpty()) {
|
||||
// Image is missing, hence no way to produce output image. (Timestamp
|
||||
// bound will be updated automatically.)
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
return helper_.RunInGlContext(
|
||||
[this, cc]() -> ::mediapipe::Status { return RenderGpu(cc); });
|
||||
#endif // !MEDIAPIPE_DISABLE_GPU
|
||||
} else {
|
||||
if (cc->Inputs().Tag("IMAGE").IsEmpty()) {
|
||||
// Image is missing, hence no way to produce output image. (Timestamp
|
||||
// bound will be updated automatically.)
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
return RenderCpu(cc);
|
||||
}
|
||||
return ::mediapipe::OkStatus();
|
||||
@@ -360,11 +428,6 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
|
||||
.Add(padding.release(), cc->InputTimestamp());
|
||||
}
|
||||
|
||||
if (cc->InputSidePackets().HasTag("ROTATION_DEGREES")) {
|
||||
rotation_ = DegreesToRotationMode(
|
||||
cc->InputSidePackets().Tag("ROTATION_DEGREES").Get<int>());
|
||||
}
|
||||
|
||||
cv::Mat rotated_mat;
|
||||
const int angle = RotationModeToDegrees(rotation_);
|
||||
cv::Point2f src_center(scaled_mat.cols / 2.0, scaled_mat.rows / 2.0);
|
||||
@@ -372,11 +435,9 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
|
||||
cv::warpAffine(scaled_mat, rotated_mat, rotation_mat, scaled_mat.size());
|
||||
|
||||
cv::Mat flipped_mat;
|
||||
if (options_.flip_horizontally() || options_.flip_vertically()) {
|
||||
if (flip_horizontally_ || flip_vertically_) {
|
||||
const int flip_code =
|
||||
options_.flip_horizontally() && options_.flip_vertically()
|
||||
? -1
|
||||
: options_.flip_horizontally();
|
||||
flip_horizontally_ && flip_vertically_ ? -1 : flip_horizontally_;
|
||||
cv::flip(rotated_mat, flipped_mat, flip_code);
|
||||
} else {
|
||||
flipped_mat = rotated_mat;
|
||||
@@ -450,11 +511,6 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
|
||||
}
|
||||
RET_CHECK(renderer) << "Unsupported input texture type";
|
||||
|
||||
if (cc->InputSidePackets().HasTag("ROTATION_DEGREES")) {
|
||||
rotation_ = DegreesToRotationMode(
|
||||
cc->InputSidePackets().Tag("ROTATION_DEGREES").Get<int>());
|
||||
}
|
||||
|
||||
mediapipe::FrameScaleMode scale_mode = mediapipe::FrameScaleModeFromProto(
|
||||
scale_mode_, mediapipe::FrameScaleMode::kStretch);
|
||||
mediapipe::FrameRotation rotation =
|
||||
@@ -469,7 +525,7 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
|
||||
|
||||
MP_RETURN_IF_ERROR(renderer->GlRender(
|
||||
src1.width(), src1.height(), dst.width(), dst.height(), scale_mode,
|
||||
rotation, options_.flip_horizontally(), options_.flip_vertically(),
|
||||
rotation, flip_horizontally_, flip_vertically_,
|
||||
/*flip_texture=*/false));
|
||||
|
||||
glActiveTexture(GL_TEXTURE1);
|
||||
|
||||
@@ -260,11 +260,11 @@ ScaleImageCalculator::~ScaleImageCalculator() {}
|
||||
&crop_width_, &crop_height_, //
|
||||
&col_start_, &row_start_));
|
||||
MP_RETURN_IF_ERROR(
|
||||
scale_image::FindOutputDimensions(crop_width_, crop_height_, //
|
||||
options_.target_width(), //
|
||||
options_.target_height(), //
|
||||
options_.preserve_aspect_ratio(), //
|
||||
options_.scale_to_multiple_of_two(), //
|
||||
scale_image::FindOutputDimensions(crop_width_, crop_height_, //
|
||||
options_.target_width(), //
|
||||
options_.target_height(), //
|
||||
options_.preserve_aspect_ratio(), //
|
||||
options_.scale_to_multiple_of(), //
|
||||
&output_width_, &output_height_));
|
||||
MP_RETURN_IF_ERROR(FindInterpolationAlgorithm(options_.algorithm(),
|
||||
&interpolation_algorithm_));
|
||||
@@ -361,17 +361,21 @@ ScaleImageCalculator::~ScaleImageCalculator() {}
|
||||
output_format_ = input_format_;
|
||||
}
|
||||
|
||||
const bool is_positive_and_even =
|
||||
(options_.scale_to_multiple_of() >= 1) &&
|
||||
(options_.scale_to_multiple_of() % 2 == 0);
|
||||
|
||||
if (output_format_ == ImageFormat::YCBCR420P) {
|
||||
RET_CHECK(options_.scale_to_multiple_of_two())
|
||||
RET_CHECK(is_positive_and_even)
|
||||
<< "ScaleImageCalculator always outputs width and height that are "
|
||||
"divisible by 2 when output format is YCbCr420P. To scale to "
|
||||
"width and height of odd numbers, the output format must be SRGB.";
|
||||
} else if (options_.preserve_aspect_ratio()) {
|
||||
RET_CHECK(options_.scale_to_multiple_of_two())
|
||||
RET_CHECK(options_.scale_to_multiple_of() == 2)
|
||||
<< "ScaleImageCalculator always outputs width and height that are "
|
||||
"divisible by 2 when perserving aspect ratio. To scale to width "
|
||||
"and height of odd numbers, please set "
|
||||
"preserve_aspect_ratio to false.";
|
||||
"divisible by 2 when preserving aspect ratio. If you'd like to "
|
||||
"set scale_to_multiple_of to something other than 2, please "
|
||||
"set preserve_aspect_ratio to false.";
|
||||
}
|
||||
|
||||
if (input_width_ > 0 && input_height_ > 0 &&
|
||||
|
||||
@@ -11,9 +11,10 @@ import "mediapipe/framework/formats/image_format.proto";
|
||||
// 2) Scale and convert the image to fit inside target_width x target_height
|
||||
// using the specified scaling algorithm. (maintaining the aspect
|
||||
// ratio if preserve_aspect_ratio is true).
|
||||
// The output width and height will be divisible by 2. It is possible to output
|
||||
// width and height that are odd number when the output format is SRGB and not
|
||||
// perserving the aspect ratio. See scale_to_multiple_of_two option for details.
|
||||
// The output width and height will be divisible by 2, by default. It is
|
||||
// possible to output width and height that are odd numbers when the output
|
||||
// format is SRGB and the aspect ratio is left unpreserved. See
|
||||
// scale_to_multiple_of for details.
|
||||
message ScaleImageCalculatorOptions {
|
||||
extend CalculatorOptions {
|
||||
optional ScaleImageCalculatorOptions ext = 66237115;
|
||||
@@ -23,7 +24,7 @@ message ScaleImageCalculatorOptions {
|
||||
// depending on the other options below. If unset, use the same width
|
||||
// or height as the input. If only one is set then determine the other
|
||||
// from the aspect ratio (after cropping). The output width and height
|
||||
// will be divisible by 2.
|
||||
// will be divisible by 2, by default.
|
||||
optional int32 target_width = 1;
|
||||
optional int32 target_height = 2;
|
||||
|
||||
@@ -31,7 +32,8 @@ message ScaleImageCalculatorOptions {
|
||||
// fits inside the box represented by target_width and target_height.
|
||||
// Otherwise it is scaled to fit target_width and target_height
|
||||
// completely. In any case, the aspect ratio that is preserved is
|
||||
// that after cropping to the minimum/maximum aspect ratio.
|
||||
// that after cropping to the minimum/maximum aspect ratio. Additionally, if
|
||||
// true, the output width and height will be divisible by 2.
|
||||
optional bool preserve_aspect_ratio = 3 [default = true];
|
||||
|
||||
// If ratio is positive, crop the image to this minimum and maximum
|
||||
@@ -95,11 +97,13 @@ message ScaleImageCalculatorOptions {
|
||||
// SRGB or YCBCR420P.
|
||||
optional ImageFormat.Format input_format = 12;
|
||||
|
||||
// If true, the output width and height will be divisible by 2. Otherwise it
|
||||
// will use the exact specified output width and height, which is only
|
||||
// supported when the output format is SRGB and preserve_aspect_ratio option
|
||||
// is set to false.
|
||||
optional bool scale_to_multiple_of_two = 13 [default = true];
|
||||
// If set to 2, the target width and height will be rounded-down
|
||||
// to the nearest even number. If set to any positive value other than 2,
|
||||
// preserve_aspect_ratio must be false and the target width and height will be
|
||||
// rounded-down to multiples of the given value. If set to any value less than
|
||||
// 1, it will be treated like 1.
|
||||
// NOTE: If set to an odd number, the output format must be SRGB.
|
||||
optional int32 scale_to_multiple_of = 13 [default = 2];
|
||||
|
||||
// If true, assume the input YUV is BT.709 (this is the HDTV standard, so most
|
||||
// content is likely using it). If false use the previous assumption of BT.601
|
||||
|
||||
@@ -88,17 +88,27 @@ double ParseRational(const std::string& rational) {
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status FindOutputDimensions(int input_width, //
|
||||
int input_height, //
|
||||
int target_width, //
|
||||
int target_height, //
|
||||
bool preserve_aspect_ratio, //
|
||||
bool scale_to_multiple_of_two, //
|
||||
::mediapipe::Status FindOutputDimensions(int input_width, //
|
||||
int input_height, //
|
||||
int target_width, //
|
||||
int target_height, //
|
||||
bool preserve_aspect_ratio, //
|
||||
int scale_to_multiple_of, //
|
||||
int* output_width,
|
||||
int* output_height) {
|
||||
CHECK(output_width);
|
||||
CHECK(output_height);
|
||||
|
||||
if (preserve_aspect_ratio) {
|
||||
RET_CHECK(scale_to_multiple_of == 2)
|
||||
<< "FindOutputDimensions always outputs width and height that are "
|
||||
"divisible by 2 when preserving aspect ratio. If you'd like to "
|
||||
"set scale_to_multiple_of to something other than 2, please "
|
||||
"set preserve_aspect_ratio to false.";
|
||||
}
|
||||
|
||||
if (scale_to_multiple_of < 1) scale_to_multiple_of = 1;
|
||||
|
||||
if (!preserve_aspect_ratio || (target_width <= 0 && target_height <= 0)) {
|
||||
if (target_width <= 0) {
|
||||
target_width = input_width;
|
||||
@@ -106,13 +116,13 @@ double ParseRational(const std::string& rational) {
|
||||
if (target_height <= 0) {
|
||||
target_height = input_height;
|
||||
}
|
||||
if (scale_to_multiple_of_two) {
|
||||
*output_width = (target_width / 2) * 2;
|
||||
*output_height = (target_height / 2) * 2;
|
||||
} else {
|
||||
*output_width = target_width;
|
||||
*output_height = target_height;
|
||||
}
|
||||
|
||||
target_width -= target_width % scale_to_multiple_of;
|
||||
target_height -= target_height % scale_to_multiple_of;
|
||||
|
||||
*output_width = target_width;
|
||||
*output_height = target_height;
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
|
||||
@@ -35,17 +35,19 @@ namespace scale_image {
|
||||
int* col_start, int* row_start);
|
||||
|
||||
// Given an input width and height, a target width and height, whether to
|
||||
// preserve the aspect ratio, and whether to round down to a multiple of 2,
|
||||
// determine the output width and height. If target_width or target_height is
|
||||
// non-positive, then they will be set to the input_width and input_height
|
||||
// respectively. The output_width and output_height will be reduced as necessary
|
||||
// to preserve_aspect_ratio and to scale_to_multipe_of_two if these options are
|
||||
// specified.
|
||||
// preserve the aspect ratio, and whether to round-down to the multiple of a
|
||||
// given number nearest to the targets, determine the output width and height.
|
||||
// If target_width or target_height is non-positive, then they will be set to
|
||||
// the input_width and input_height respectively. If scale_to_multiple_of is
|
||||
// less than 1, it will be treated like 1. The output_width and
|
||||
// output_height will be reduced as necessary to preserve_aspect_ratio if the
|
||||
// option is specified. If preserving the aspect ratio is desired, you must set
|
||||
// scale_to_multiple_of to 2.
|
||||
::mediapipe::Status FindOutputDimensions(int input_width, int input_height, //
|
||||
int target_width,
|
||||
int target_height, //
|
||||
bool preserve_aspect_ratio, //
|
||||
bool scale_to_multiple_of_two, //
|
||||
int target_height, //
|
||||
bool preserve_aspect_ratio, //
|
||||
int scale_to_multiple_of, //
|
||||
int* output_width, int* output_height);
|
||||
|
||||
} // namespace scale_image
|
||||
|
||||
@@ -79,49 +79,49 @@ TEST(ScaleImageUtilsTest, FindOutputDimensionsPreserveRatio) {
|
||||
int output_width;
|
||||
int output_height;
|
||||
// Not scale.
|
||||
MP_ASSERT_OK(FindOutputDimensions(200, 100, -1, -1, true, true, &output_width,
|
||||
MP_ASSERT_OK(FindOutputDimensions(200, 100, -1, -1, true, 2, &output_width,
|
||||
&output_height));
|
||||
EXPECT_EQ(200, output_width);
|
||||
EXPECT_EQ(100, output_height);
|
||||
// Not scale with odd input size.
|
||||
MP_ASSERT_OK(FindOutputDimensions(201, 101, -1, -1, false, false,
|
||||
&output_width, &output_height));
|
||||
MP_ASSERT_OK(FindOutputDimensions(201, 101, -1, -1, false, 1, &output_width,
|
||||
&output_height));
|
||||
EXPECT_EQ(201, output_width);
|
||||
EXPECT_EQ(101, output_height);
|
||||
// Scale down by 1/2.
|
||||
MP_ASSERT_OK(FindOutputDimensions(200, 100, 100, -1, true, true,
|
||||
&output_width, &output_height));
|
||||
MP_ASSERT_OK(FindOutputDimensions(200, 100, 100, -1, true, 2, &output_width,
|
||||
&output_height));
|
||||
EXPECT_EQ(100, output_width);
|
||||
EXPECT_EQ(50, output_height);
|
||||
// Scale up, doubling dimensions.
|
||||
MP_ASSERT_OK(FindOutputDimensions(200, 100, -1, 200, true, true,
|
||||
&output_width, &output_height));
|
||||
MP_ASSERT_OK(FindOutputDimensions(200, 100, -1, 200, true, 2, &output_width,
|
||||
&output_height));
|
||||
EXPECT_EQ(400, output_width);
|
||||
EXPECT_EQ(200, output_height);
|
||||
// Fits a 2:1 image into a 150 x 150 box. Output dimensions are always
|
||||
// visible by 2.
|
||||
MP_ASSERT_OK(FindOutputDimensions(200, 100, 150, 150, true, true,
|
||||
&output_width, &output_height));
|
||||
MP_ASSERT_OK(FindOutputDimensions(200, 100, 150, 150, true, 2, &output_width,
|
||||
&output_height));
|
||||
EXPECT_EQ(150, output_width);
|
||||
EXPECT_EQ(74, output_height);
|
||||
// Fits a 2:1 image into a 400 x 50 box.
|
||||
MP_ASSERT_OK(FindOutputDimensions(200, 100, 400, 50, true, true,
|
||||
&output_width, &output_height));
|
||||
MP_ASSERT_OK(FindOutputDimensions(200, 100, 400, 50, true, 2, &output_width,
|
||||
&output_height));
|
||||
EXPECT_EQ(100, output_width);
|
||||
EXPECT_EQ(50, output_height);
|
||||
// Scale to multiple number with odd targe size.
|
||||
MP_ASSERT_OK(FindOutputDimensions(200, 100, 101, -1, true, true,
|
||||
&output_width, &output_height));
|
||||
MP_ASSERT_OK(FindOutputDimensions(200, 100, 101, -1, true, 2, &output_width,
|
||||
&output_height));
|
||||
EXPECT_EQ(100, output_width);
|
||||
EXPECT_EQ(50, output_height);
|
||||
// Scale to multiple number with odd targe size.
|
||||
MP_ASSERT_OK(FindOutputDimensions(200, 100, 101, -1, true, false,
|
||||
&output_width, &output_height));
|
||||
MP_ASSERT_OK(FindOutputDimensions(200, 100, 101, -1, true, 2, &output_width,
|
||||
&output_height));
|
||||
EXPECT_EQ(100, output_width);
|
||||
EXPECT_EQ(50, output_height);
|
||||
// Scale to odd size.
|
||||
MP_ASSERT_OK(FindOutputDimensions(200, 100, 151, 101, false, false,
|
||||
&output_width, &output_height));
|
||||
MP_ASSERT_OK(FindOutputDimensions(200, 100, 151, 101, false, 1, &output_width,
|
||||
&output_height));
|
||||
EXPECT_EQ(151, output_width);
|
||||
EXPECT_EQ(101, output_height);
|
||||
}
|
||||
@@ -131,22 +131,62 @@ TEST(ScaleImageUtilsTest, FindOutputDimensionsNoAspectRatio) {
|
||||
int output_width;
|
||||
int output_height;
|
||||
// Scale width only.
|
||||
MP_ASSERT_OK(FindOutputDimensions(200, 100, 100, -1, false, true,
|
||||
&output_width, &output_height));
|
||||
MP_ASSERT_OK(FindOutputDimensions(200, 100, 100, -1, false, 2, &output_width,
|
||||
&output_height));
|
||||
EXPECT_EQ(100, output_width);
|
||||
EXPECT_EQ(100, output_height);
|
||||
// Scale height only.
|
||||
MP_ASSERT_OK(FindOutputDimensions(200, 100, -1, 200, false, true,
|
||||
&output_width, &output_height));
|
||||
MP_ASSERT_OK(FindOutputDimensions(200, 100, -1, 200, false, 2, &output_width,
|
||||
&output_height));
|
||||
EXPECT_EQ(200, output_width);
|
||||
EXPECT_EQ(200, output_height);
|
||||
// Scale both dimensions.
|
||||
MP_ASSERT_OK(FindOutputDimensions(200, 100, 150, 200, false, true,
|
||||
&output_width, &output_height));
|
||||
MP_ASSERT_OK(FindOutputDimensions(200, 100, 150, 200, false, 2, &output_width,
|
||||
&output_height));
|
||||
EXPECT_EQ(150, output_width);
|
||||
EXPECT_EQ(200, output_height);
|
||||
}
|
||||
|
||||
// Tests scale_to_multiple_of.
|
||||
TEST(ScaleImageUtilsTest, FindOutputDimensionsDownScaleToMultipleOf) {
|
||||
int output_width;
|
||||
int output_height;
|
||||
// Set no targets, downscale to a multiple of 8.
|
||||
MP_ASSERT_OK(FindOutputDimensions(100, 100, -1, -1, false, 8, &output_width,
|
||||
&output_height));
|
||||
EXPECT_EQ(96, output_width);
|
||||
EXPECT_EQ(96, output_height);
|
||||
// Set width target, downscale to a multiple of 8.
|
||||
MP_ASSERT_OK(FindOutputDimensions(200, 100, 100, -1, false, 8, &output_width,
|
||||
&output_height));
|
||||
EXPECT_EQ(96, output_width);
|
||||
EXPECT_EQ(96, output_height);
|
||||
// Set height target, downscale to a multiple of 8.
|
||||
MP_ASSERT_OK(FindOutputDimensions(201, 101, -1, 201, false, 8, &output_width,
|
||||
&output_height));
|
||||
EXPECT_EQ(200, output_width);
|
||||
EXPECT_EQ(200, output_height);
|
||||
// Set both targets, downscale to a multiple of 8.
|
||||
MP_ASSERT_OK(FindOutputDimensions(200, 100, 150, 200, false, 8, &output_width,
|
||||
&output_height));
|
||||
EXPECT_EQ(144, output_width);
|
||||
EXPECT_EQ(200, output_height);
|
||||
// Doesn't throw error if keep aspect is true and downscale multiple is 2.
|
||||
MP_ASSERT_OK(FindOutputDimensions(200, 100, 400, 200, true, 2, &output_width,
|
||||
&output_height));
|
||||
EXPECT_EQ(400, output_width);
|
||||
EXPECT_EQ(200, output_height);
|
||||
// Throws error if keep aspect is true, but downscale multiple is not 2.
|
||||
ASSERT_THAT(FindOutputDimensions(200, 100, 400, 200, true, 4, &output_width,
|
||||
&output_height),
|
||||
testing::Not(testing::status::IsOk()));
|
||||
// Downscaling to multiple ignored if multiple is less than 2.
|
||||
MP_ASSERT_OK(FindOutputDimensions(200, 100, 401, 201, false, 1, &output_width,
|
||||
&output_height));
|
||||
EXPECT_EQ(401, output_width);
|
||||
EXPECT_EQ(201, output_height);
|
||||
}
|
||||
|
||||
} // namespace
|
||||
} // namespace scale_image
|
||||
} // namespace mediapipe
|
||||
|
||||
@@ -138,7 +138,7 @@ mediapipe_cc_proto_library(
|
||||
srcs = ["image_frame_to_tensor_calculator.proto"],
|
||||
cc_deps = [
|
||||
"//mediapipe/framework:calculator_cc_proto",
|
||||
"@org_tensorflow//tensorflow/core:protos_all",
|
||||
"@org_tensorflow//tensorflow/core:protos_all_cc",
|
||||
],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":image_frame_to_tensor_calculator_proto"],
|
||||
@@ -173,7 +173,7 @@ mediapipe_cc_proto_library(
|
||||
srcs = ["pack_media_sequence_calculator.proto"],
|
||||
cc_deps = [
|
||||
"//mediapipe/framework:calculator_cc_proto",
|
||||
"@org_tensorflow//tensorflow/core:protos_all",
|
||||
"@org_tensorflow//tensorflow/core:protos_all_cc",
|
||||
],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":pack_media_sequence_calculator_proto"],
|
||||
@@ -192,7 +192,7 @@ mediapipe_cc_proto_library(
|
||||
srcs = ["tensorflow_session_from_frozen_graph_generator.proto"],
|
||||
cc_deps = [
|
||||
"//mediapipe/framework:packet_generator_cc_proto",
|
||||
"@org_tensorflow//tensorflow/core:protos_all",
|
||||
"@org_tensorflow//tensorflow/core:protos_all_cc",
|
||||
],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":tensorflow_session_from_frozen_graph_generator_proto"],
|
||||
@@ -203,7 +203,7 @@ mediapipe_cc_proto_library(
|
||||
srcs = ["tensorflow_session_from_frozen_graph_calculator.proto"],
|
||||
cc_deps = [
|
||||
"//mediapipe/framework:calculator_cc_proto",
|
||||
"@org_tensorflow//tensorflow/core:protos_all",
|
||||
"@org_tensorflow//tensorflow/core:protos_all_cc",
|
||||
],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":tensorflow_session_from_frozen_graph_calculator_proto"],
|
||||
@@ -277,7 +277,7 @@ mediapipe_cc_proto_library(
|
||||
srcs = ["vector_int_to_tensor_calculator_options.proto"],
|
||||
cc_deps = [
|
||||
"//mediapipe/framework:calculator_cc_proto",
|
||||
"@org_tensorflow//tensorflow/core:protos_all",
|
||||
"@org_tensorflow//tensorflow/core:protos_all_cc",
|
||||
],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [":vector_int_to_tensor_calculator_options_proto"],
|
||||
@@ -408,7 +408,7 @@ cc_library(
|
||||
"//mediapipe/util/sequence:media_sequence",
|
||||
"//mediapipe/util/sequence:media_sequence_util",
|
||||
"@com_google_absl//absl/strings",
|
||||
"@org_tensorflow//tensorflow/core:protos_all",
|
||||
"@org_tensorflow//tensorflow/core:protos_all_cc",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
@@ -423,7 +423,7 @@ cc_library(
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/port:status",
|
||||
"@org_tensorflow//tensorflow/core:protos_all",
|
||||
"@org_tensorflow//tensorflow/core:protos_all_cc",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
@@ -654,7 +654,7 @@ cc_library(
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/port:status",
|
||||
"@org_tensorflow//tensorflow/core:lib",
|
||||
"@org_tensorflow//tensorflow/core:protos_all",
|
||||
"@org_tensorflow//tensorflow/core:protos_all_cc",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
@@ -695,7 +695,7 @@ cc_library(
|
||||
"//mediapipe/util:audio_decoder_cc_proto",
|
||||
"//mediapipe/util/sequence:media_sequence",
|
||||
"@com_google_absl//absl/strings",
|
||||
"@org_tensorflow//tensorflow/core:protos_all",
|
||||
"@org_tensorflow//tensorflow/core:protos_all_cc",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
@@ -737,7 +737,7 @@ cc_library(
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:packet",
|
||||
"//mediapipe/framework/port:status",
|
||||
"@org_tensorflow//tensorflow/core:protos_all",
|
||||
"@org_tensorflow//tensorflow/core:protos_all_cc",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
@@ -745,6 +745,7 @@ cc_library(
|
||||
cc_test(
|
||||
name = "graph_tensors_packet_generator_test",
|
||||
srcs = ["graph_tensors_packet_generator_test.cc"],
|
||||
linkstatic = 1,
|
||||
deps = [
|
||||
":graph_tensors_packet_generator",
|
||||
":graph_tensors_packet_generator_cc_proto",
|
||||
@@ -761,6 +762,7 @@ cc_test(
|
||||
name = "image_frame_to_tensor_calculator_test",
|
||||
size = "small",
|
||||
srcs = ["image_frame_to_tensor_calculator_test.cc"],
|
||||
linkstatic = 1,
|
||||
deps = [
|
||||
":image_frame_to_tensor_calculator",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
@@ -777,6 +779,7 @@ cc_test(
|
||||
name = "matrix_to_tensor_calculator_test",
|
||||
size = "small",
|
||||
srcs = ["matrix_to_tensor_calculator_test.cc"],
|
||||
linkstatic = 1,
|
||||
deps = [
|
||||
":matrix_to_tensor_calculator",
|
||||
":matrix_to_tensor_calculator_options_cc_proto",
|
||||
@@ -793,6 +796,7 @@ cc_test(
|
||||
name = "lapped_tensor_buffer_calculator_test",
|
||||
size = "small",
|
||||
srcs = ["lapped_tensor_buffer_calculator_test.cc"],
|
||||
linkstatic = 1,
|
||||
deps = [
|
||||
":lapped_tensor_buffer_calculator",
|
||||
":lapped_tensor_buffer_calculator_cc_proto",
|
||||
@@ -801,7 +805,7 @@ cc_test(
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"@com_google_absl//absl/memory",
|
||||
"@org_tensorflow//tensorflow/core:framework",
|
||||
"@org_tensorflow//tensorflow/core:protos_all",
|
||||
"@org_tensorflow//tensorflow/core:protos_all_cc",
|
||||
],
|
||||
)
|
||||
|
||||
@@ -840,7 +844,7 @@ cc_test(
|
||||
"//mediapipe/util/sequence:media_sequence",
|
||||
"@com_google_absl//absl/memory",
|
||||
"@com_google_absl//absl/strings",
|
||||
"@org_tensorflow//tensorflow/core:protos_all",
|
||||
"@org_tensorflow//tensorflow/core:protos_all_cc",
|
||||
],
|
||||
)
|
||||
|
||||
@@ -867,7 +871,7 @@ cc_test(
|
||||
"@com_google_absl//absl/strings",
|
||||
"@org_tensorflow//tensorflow/core:direct_session",
|
||||
"@org_tensorflow//tensorflow/core:framework",
|
||||
"@org_tensorflow//tensorflow/core:protos_all",
|
||||
"@org_tensorflow//tensorflow/core:protos_all_cc",
|
||||
"@org_tensorflow//tensorflow/core:testlib",
|
||||
"@org_tensorflow//tensorflow/core/kernels:conv_ops",
|
||||
"@org_tensorflow//tensorflow/core/kernels:math",
|
||||
@@ -897,7 +901,7 @@ cc_test(
|
||||
"@com_google_absl//absl/strings",
|
||||
"@org_tensorflow//tensorflow/core:direct_session",
|
||||
"@org_tensorflow//tensorflow/core:framework",
|
||||
"@org_tensorflow//tensorflow/core:protos_all",
|
||||
"@org_tensorflow//tensorflow/core:protos_all_cc",
|
||||
"@org_tensorflow//tensorflow/core:testlib",
|
||||
"@org_tensorflow//tensorflow/core/kernels:conv_ops",
|
||||
"@org_tensorflow//tensorflow/core/kernels:math",
|
||||
@@ -956,6 +960,7 @@ cc_test(
|
||||
cc_test(
|
||||
name = "tensor_squeeze_dimensions_calculator_test",
|
||||
srcs = ["tensor_squeeze_dimensions_calculator_test.cc"],
|
||||
linkstatic = 1,
|
||||
deps = [
|
||||
":tensor_squeeze_dimensions_calculator",
|
||||
":tensor_squeeze_dimensions_calculator_cc_proto",
|
||||
@@ -963,7 +968,7 @@ cc_test(
|
||||
"//mediapipe/framework:calculator_runner",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"@org_tensorflow//tensorflow/core:framework",
|
||||
"@org_tensorflow//tensorflow/core:protos_all",
|
||||
"@org_tensorflow//tensorflow/core:protos_all_cc",
|
||||
],
|
||||
)
|
||||
|
||||
@@ -971,6 +976,7 @@ cc_test(
|
||||
name = "tensor_to_image_frame_calculator_test",
|
||||
size = "small",
|
||||
srcs = ["tensor_to_image_frame_calculator_test.cc"],
|
||||
linkstatic = 1,
|
||||
deps = [
|
||||
":tensor_to_image_frame_calculator",
|
||||
":tensor_to_image_frame_calculator_cc_proto",
|
||||
@@ -979,7 +985,7 @@ cc_test(
|
||||
"//mediapipe/framework/formats:image_frame",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"@org_tensorflow//tensorflow/core:framework",
|
||||
"@org_tensorflow//tensorflow/core:protos_all",
|
||||
"@org_tensorflow//tensorflow/core:protos_all_cc",
|
||||
],
|
||||
)
|
||||
|
||||
@@ -987,6 +993,7 @@ cc_test(
|
||||
name = "tensor_to_matrix_calculator_test",
|
||||
size = "small",
|
||||
srcs = ["tensor_to_matrix_calculator_test.cc"],
|
||||
linkstatic = 1,
|
||||
deps = [
|
||||
":tensor_to_matrix_calculator",
|
||||
":tensor_to_matrix_calculator_cc_proto",
|
||||
@@ -996,13 +1003,14 @@ cc_test(
|
||||
"//mediapipe/framework/formats:time_series_header_cc_proto",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"@org_tensorflow//tensorflow/core:framework",
|
||||
"@org_tensorflow//tensorflow/core:protos_all",
|
||||
"@org_tensorflow//tensorflow/core:protos_all_cc",
|
||||
],
|
||||
)
|
||||
|
||||
cc_test(
|
||||
name = "tensor_to_vector_float_calculator_test",
|
||||
srcs = ["tensor_to_vector_float_calculator_test.cc"],
|
||||
linkstatic = 1,
|
||||
deps = [
|
||||
":tensor_to_vector_float_calculator",
|
||||
":tensor_to_vector_float_calculator_options_cc_proto",
|
||||
@@ -1010,7 +1018,7 @@ cc_test(
|
||||
"//mediapipe/framework:calculator_runner",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"@org_tensorflow//tensorflow/core:framework",
|
||||
"@org_tensorflow//tensorflow/core:protos_all",
|
||||
"@org_tensorflow//tensorflow/core:protos_all_cc",
|
||||
],
|
||||
)
|
||||
|
||||
@@ -1030,13 +1038,14 @@ cc_test(
|
||||
"//mediapipe/util/sequence:media_sequence",
|
||||
"@com_google_absl//absl/memory",
|
||||
"@com_google_absl//absl/strings",
|
||||
"@org_tensorflow//tensorflow/core:protos_all",
|
||||
"@org_tensorflow//tensorflow/core:protos_all_cc",
|
||||
],
|
||||
)
|
||||
|
||||
cc_test(
|
||||
name = "vector_int_to_tensor_calculator_test",
|
||||
srcs = ["vector_int_to_tensor_calculator_test.cc"],
|
||||
linkstatic = 1,
|
||||
deps = [
|
||||
":vector_int_to_tensor_calculator",
|
||||
":vector_int_to_tensor_calculator_options_cc_proto",
|
||||
@@ -1044,13 +1053,14 @@ cc_test(
|
||||
"//mediapipe/framework:calculator_runner",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"@org_tensorflow//tensorflow/core:framework",
|
||||
"@org_tensorflow//tensorflow/core:protos_all",
|
||||
"@org_tensorflow//tensorflow/core:protos_all_cc",
|
||||
],
|
||||
)
|
||||
|
||||
cc_test(
|
||||
name = "vector_float_to_tensor_calculator_test",
|
||||
srcs = ["vector_float_to_tensor_calculator_test.cc"],
|
||||
linkstatic = 1,
|
||||
deps = [
|
||||
":vector_float_to_tensor_calculator",
|
||||
":vector_float_to_tensor_calculator_options_cc_proto",
|
||||
@@ -1058,7 +1068,7 @@ cc_test(
|
||||
"//mediapipe/framework:calculator_runner",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"@org_tensorflow//tensorflow/core:framework",
|
||||
"@org_tensorflow//tensorflow/core:protos_all",
|
||||
"@org_tensorflow//tensorflow/core:protos_all_cc",
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
#if !defined(__ANDROID__)
|
||||
#include "mediapipe/framework/port/file_helpers.h"
|
||||
#endif
|
||||
#include "absl/strings/substitute.h"
|
||||
#include "absl/strings/str_replace.h"
|
||||
#include "mediapipe/calculators/tensorflow/tensorflow_session.h"
|
||||
#include "mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator.pb.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
@@ -63,7 +63,7 @@ const std::string MaybeConvertSignatureToTag(
|
||||
output.resize(name.length());
|
||||
std::transform(name.begin(), name.end(), output.begin(),
|
||||
[](unsigned char c) { return std::toupper(c); });
|
||||
output = absl::Substitute(output, "/", "_");
|
||||
output = absl::StrReplaceAll(output, {{"/", "_"}});
|
||||
return output;
|
||||
} else {
|
||||
return name;
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "absl/strings/substitute.h"
|
||||
#include "absl/strings/str_replace.h"
|
||||
#include "mediapipe/calculators/tensorflow/tensorflow_session.h"
|
||||
#include "mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator.pb.h"
|
||||
#include "mediapipe/framework/calculator.pb.h"
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
#if !defined(__ANDROID__)
|
||||
#include "mediapipe/framework/port/file_helpers.h"
|
||||
#endif
|
||||
#include "absl/strings/substitute.h"
|
||||
#include "absl/strings/str_replace.h"
|
||||
#include "mediapipe/calculators/tensorflow/tensorflow_session.h"
|
||||
#include "mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator.pb.h"
|
||||
#include "mediapipe/framework/deps/file_path.h"
|
||||
@@ -65,7 +65,7 @@ const std::string MaybeConvertSignatureToTag(
|
||||
output.resize(name.length());
|
||||
std::transform(name.begin(), name.end(), output.begin(),
|
||||
[](unsigned char c) { return std::toupper(c); });
|
||||
output = absl::Substitute(output, "/", "_");
|
||||
output = absl::StrReplaceAll(output, {{"/", "_"}});
|
||||
return output;
|
||||
} else {
|
||||
return name;
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "absl/strings/substitute.h"
|
||||
#include "absl/strings/str_replace.h"
|
||||
#include "mediapipe/calculators/tensorflow/tensorflow_session.h"
|
||||
#include "mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator.pb.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
|
||||
@@ -485,6 +485,7 @@ cc_test(
|
||||
"//mediapipe/framework/port:integral_types",
|
||||
"//mediapipe/framework/port:parse_text_proto",
|
||||
"//mediapipe/framework/tool:validate_type",
|
||||
"@com_google_absl//absl/strings",
|
||||
"@org_tensorflow//tensorflow/lite:framework",
|
||||
"@org_tensorflow//tensorflow/lite/kernels:builtin_ops",
|
||||
],
|
||||
|
||||
@@ -148,7 +148,7 @@ struct GPUData {
|
||||
// options: {
|
||||
// [mediapipe.TfLiteInferenceCalculatorOptions.ext] {
|
||||
// model_path: "modelname.tflite"
|
||||
// use_gpu: true
|
||||
// delegate { gpu {} }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
@@ -163,6 +163,9 @@ struct GPUData {
|
||||
//
|
||||
class TfLiteInferenceCalculator : public CalculatorBase {
|
||||
public:
|
||||
using TfLiteDelegatePtr =
|
||||
std::unique_ptr<TfLiteDelegate, std::function<void(TfLiteDelegate*)>>;
|
||||
|
||||
static ::mediapipe::Status GetContract(CalculatorContract* cc);
|
||||
|
||||
::mediapipe::Status Open(CalculatorContext* cc) override;
|
||||
@@ -176,7 +179,7 @@ class TfLiteInferenceCalculator : public CalculatorBase {
|
||||
|
||||
std::unique_ptr<tflite::Interpreter> interpreter_;
|
||||
std::unique_ptr<tflite::FlatBufferModel> model_;
|
||||
TfLiteDelegate* delegate_ = nullptr;
|
||||
TfLiteDelegatePtr delegate_;
|
||||
|
||||
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
|
||||
mediapipe::GlCalculatorHelper gpu_helper_;
|
||||
@@ -212,12 +215,18 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
|
||||
RET_CHECK(cc->Outputs().HasTag("TENSORS") ^
|
||||
cc->Outputs().HasTag("TENSORS_GPU"));
|
||||
|
||||
bool use_gpu = false;
|
||||
const auto& options =
|
||||
cc->Options<::mediapipe::TfLiteInferenceCalculatorOptions>();
|
||||
bool use_gpu =
|
||||
options.has_delegate() ? options.delegate().has_gpu() : options.use_gpu();
|
||||
|
||||
if (cc->Inputs().HasTag("TENSORS"))
|
||||
cc->Inputs().Tag("TENSORS").Set<std::vector<TfLiteTensor>>();
|
||||
#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__)
|
||||
if (cc->Inputs().HasTag("TENSORS_GPU")) {
|
||||
RET_CHECK(!options.has_delegate() || options.delegate().has_gpu())
|
||||
<< "GPU input is compatible with GPU delegate only.";
|
||||
|
||||
cc->Inputs().Tag("TENSORS_GPU").Set<std::vector<GpuTensor>>();
|
||||
use_gpu |= true;
|
||||
}
|
||||
@@ -227,6 +236,9 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
|
||||
cc->Outputs().Tag("TENSORS").Set<std::vector<TfLiteTensor>>();
|
||||
#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__)
|
||||
if (cc->Outputs().HasTag("TENSORS_GPU")) {
|
||||
RET_CHECK(!options.has_delegate() || options.delegate().has_gpu())
|
||||
<< "GPU output is compatible with GPU delegate only.";
|
||||
|
||||
cc->Outputs().Tag("TENSORS_GPU").Set<std::vector<GpuTensor>>();
|
||||
use_gpu |= true;
|
||||
}
|
||||
@@ -238,10 +250,6 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
|
||||
.Set<tflite::ops::builtin::BuiltinOpResolver>();
|
||||
}
|
||||
|
||||
const auto& options =
|
||||
cc->Options<::mediapipe::TfLiteInferenceCalculatorOptions>();
|
||||
use_gpu |= options.use_gpu();
|
||||
|
||||
if (use_gpu) {
|
||||
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
|
||||
MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc));
|
||||
@@ -454,7 +462,7 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
|
||||
if (gpu_inference_) {
|
||||
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
|
||||
MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext([this]() -> Status {
|
||||
TfLiteGpuDelegateDelete(delegate_);
|
||||
delegate_ = nullptr;
|
||||
for (int i = 0; i < gpu_data_in_.size(); ++i) {
|
||||
gpu_data_in_[i].reset();
|
||||
}
|
||||
@@ -464,7 +472,7 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
|
||||
return ::mediapipe::OkStatus();
|
||||
}));
|
||||
#elif defined(MEDIAPIPE_IOS)
|
||||
TFLGpuDelegateDelete(delegate_);
|
||||
delegate_ = nullptr;
|
||||
for (int i = 0; i < gpu_data_in_.size(); ++i) {
|
||||
gpu_data_in_[i].reset();
|
||||
}
|
||||
@@ -472,8 +480,9 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
|
||||
gpu_data_out_[i].reset();
|
||||
}
|
||||
#endif
|
||||
} else {
|
||||
delegate_ = nullptr;
|
||||
}
|
||||
delegate_ = nullptr;
|
||||
}
|
||||
#if defined(MEDIAPIPE_EDGE_TPU)
|
||||
edgetpu_context_.reset();
|
||||
@@ -501,7 +510,8 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
|
||||
}
|
||||
|
||||
// Get execution modes.
|
||||
gpu_inference_ = options.use_gpu();
|
||||
gpu_inference_ =
|
||||
options.has_delegate() ? options.delegate().has_gpu() : options.use_gpu();
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
@@ -526,8 +536,12 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
|
||||
|
||||
RET_CHECK(interpreter_);
|
||||
|
||||
#if defined(__EMSCRIPTEN__)
|
||||
#if defined(__EMSCRIPTEN__) || defined(MEDIAPIPE_EDGE_TPU)
|
||||
interpreter_->SetNumThreads(1);
|
||||
#else
|
||||
interpreter_->SetNumThreads(
|
||||
cc->Options<mediapipe::TfLiteInferenceCalculatorOptions>()
|
||||
.cpu_num_thread());
|
||||
#endif // __EMSCRIPTEN__
|
||||
|
||||
if (gpu_output_) {
|
||||
@@ -545,20 +559,37 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
|
||||
|
||||
::mediapipe::Status TfLiteInferenceCalculator::LoadDelegate(
|
||||
CalculatorContext* cc) {
|
||||
#if defined(MEDIAPIPE_ANDROID)
|
||||
const auto& calculator_opts =
|
||||
cc->Options<mediapipe::TfLiteInferenceCalculatorOptions>();
|
||||
if (calculator_opts.has_delegate() &&
|
||||
calculator_opts.delegate().has_tflite()) {
|
||||
// Default tflite inference requeqsted - no need to modify graph.
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
if (!gpu_inference_) {
|
||||
if (cc->Options<mediapipe::TfLiteInferenceCalculatorOptions>()
|
||||
.use_nnapi()) {
|
||||
#if defined(MEDIAPIPE_ANDROID)
|
||||
const bool nnapi_requested = calculator_opts.has_delegate()
|
||||
? calculator_opts.delegate().has_nnapi()
|
||||
: calculator_opts.use_nnapi();
|
||||
if (nnapi_requested) {
|
||||
// Attempt to use NNAPI.
|
||||
// If not supported, the default CPU delegate will be created and used.
|
||||
interpreter_->SetAllowFp16PrecisionForFp32(1);
|
||||
delegate_ = tflite::NnApiDelegate();
|
||||
RET_CHECK_EQ(interpreter_->ModifyGraphWithDelegate(delegate_), kTfLiteOk);
|
||||
delegate_ =
|
||||
TfLiteDelegatePtr(tflite::NnApiDelegate(), [](TfLiteDelegate*) {
|
||||
// No need to free according to tflite::NnApiDelegate()
|
||||
// documentation.
|
||||
});
|
||||
RET_CHECK_EQ(interpreter_->ModifyGraphWithDelegate(delegate_.get()),
|
||||
kTfLiteOk);
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
#endif // MEDIAPIPE_ANDROID
|
||||
|
||||
// Return, no need for GPU delegate below.
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
#endif // ANDROID
|
||||
|
||||
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
|
||||
// Configure and create the delegate.
|
||||
@@ -568,7 +599,9 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
|
||||
TFLITE_GL_OBJECT_TYPE_FASTEST;
|
||||
options.compile_options.dynamic_batch_enabled = 0;
|
||||
options.compile_options.inline_parameters = 1;
|
||||
if (!delegate_) delegate_ = TfLiteGpuDelegateCreate(&options);
|
||||
if (!delegate_)
|
||||
delegate_ = TfLiteDelegatePtr(TfLiteGpuDelegateCreate(&options),
|
||||
&TfLiteGpuDelegateDelete);
|
||||
|
||||
if (gpu_input_) {
|
||||
// Get input image sizes.
|
||||
@@ -586,7 +619,7 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
|
||||
::tflite::gpu::gl::CreateReadWriteShaderStorageBuffer<float>(
|
||||
gpu_data_in_[i]->elements, &gpu_data_in_[i]->buffer));
|
||||
RET_CHECK_EQ(TfLiteGpuDelegateBindBufferToTensor(
|
||||
delegate_, gpu_data_in_[i]->buffer.id(),
|
||||
delegate_.get(), gpu_data_in_[i]->buffer.id(),
|
||||
interpreter_->inputs()[i]),
|
||||
kTfLiteOk);
|
||||
}
|
||||
@@ -609,15 +642,16 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
|
||||
for (int i = 0; i < gpu_data_out_.size(); ++i) {
|
||||
RET_CHECK_CALL(CreateReadWriteShaderStorageBuffer<float>(
|
||||
gpu_data_out_[i]->elements, &gpu_data_out_[i]->buffer));
|
||||
RET_CHECK_EQ(
|
||||
TfLiteGpuDelegateBindBufferToTensor(
|
||||
delegate_, gpu_data_out_[i]->buffer.id(), output_indices[i]),
|
||||
kTfLiteOk);
|
||||
RET_CHECK_EQ(TfLiteGpuDelegateBindBufferToTensor(
|
||||
delegate_.get(), gpu_data_out_[i]->buffer.id(),
|
||||
output_indices[i]),
|
||||
kTfLiteOk);
|
||||
}
|
||||
}
|
||||
|
||||
// Must call this last.
|
||||
RET_CHECK_EQ(interpreter_->ModifyGraphWithDelegate(delegate_), kTfLiteOk);
|
||||
RET_CHECK_EQ(interpreter_->ModifyGraphWithDelegate(delegate_.get()),
|
||||
kTfLiteOk);
|
||||
#endif // OpenGL
|
||||
|
||||
#if defined(MEDIAPIPE_IOS)
|
||||
@@ -626,7 +660,9 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
|
||||
TFLGpuDelegateOptions options;
|
||||
options.allow_precision_loss = true;
|
||||
options.wait_type = TFLGpuDelegateWaitType::TFLGpuDelegateWaitTypePassive;
|
||||
if (!delegate_) delegate_ = TFLGpuDelegateCreate(&options);
|
||||
if (!delegate_)
|
||||
delegate_ = TfLiteDelegatePtr(TFLGpuDelegateCreate(&options),
|
||||
&TFLGpuDelegateDelete);
|
||||
id<MTLDevice> device = gpu_helper_.mtlDevice;
|
||||
|
||||
if (gpu_input_) {
|
||||
@@ -678,10 +714,12 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
|
||||
gpu_data_in_[i]->buffer =
|
||||
[device newBufferWithLength:gpu_data_in_[i]->elements * kHalfSize
|
||||
options:MTLResourceStorageModeShared];
|
||||
RET_CHECK_EQ(interpreter_->ModifyGraphWithDelegate(delegate_), kTfLiteOk);
|
||||
RET_CHECK_EQ(TFLGpuDelegateBindMetalBufferToTensor(
|
||||
delegate_, input_indices[i], gpu_data_in_[i]->buffer),
|
||||
true);
|
||||
RET_CHECK_EQ(interpreter_->ModifyGraphWithDelegate(delegate_.get()),
|
||||
kTfLiteOk);
|
||||
RET_CHECK_EQ(
|
||||
TFLGpuDelegateBindMetalBufferToTensor(
|
||||
delegate_.get(), input_indices[i], gpu_data_in_[i]->buffer),
|
||||
true);
|
||||
}
|
||||
}
|
||||
if (gpu_output_) {
|
||||
@@ -725,9 +763,10 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
|
||||
gpu_data_out_[i]->buffer =
|
||||
[device newBufferWithLength:gpu_data_out_[i]->elements * kHalfSize
|
||||
options:MTLResourceStorageModeShared];
|
||||
RET_CHECK_EQ(TFLGpuDelegateBindMetalBufferToTensor(
|
||||
delegate_, output_indices[i], gpu_data_out_[i]->buffer),
|
||||
true);
|
||||
RET_CHECK_EQ(
|
||||
TFLGpuDelegateBindMetalBufferToTensor(
|
||||
delegate_.get(), output_indices[i], gpu_data_out_[i]->buffer),
|
||||
true);
|
||||
}
|
||||
|
||||
// Create converter for GPU output.
|
||||
|
||||
@@ -27,7 +27,7 @@ import "mediapipe/framework/calculator.proto";
|
||||
// options {
|
||||
// [mediapipe.TfLiteInferenceCalculatorOptions.ext] {
|
||||
// model_path: "model.tflite"
|
||||
// use_gpu: true
|
||||
// delegate { gpu {} }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
@@ -37,6 +37,22 @@ message TfLiteInferenceCalculatorOptions {
|
||||
optional TfLiteInferenceCalculatorOptions ext = 233867213;
|
||||
}
|
||||
|
||||
message Delegate {
|
||||
// Default inference provided by tflite.
|
||||
message TfLite {}
|
||||
// Delegate to run GPU inference depending on the device.
|
||||
// (Can use OpenGl, OpenCl, Metal depending on the device.)
|
||||
message Gpu {}
|
||||
// Android only.
|
||||
message Nnapi {}
|
||||
|
||||
oneof delegate {
|
||||
TfLite tflite = 1;
|
||||
Gpu gpu = 2;
|
||||
Nnapi nnapi = 3;
|
||||
}
|
||||
}
|
||||
|
||||
// Path to the TF Lite model (ex: /path/to/modelname.tflite).
|
||||
// On mobile, this is generally just modelname.tflite.
|
||||
optional string model_path = 1;
|
||||
@@ -44,10 +60,22 @@ message TfLiteInferenceCalculatorOptions {
|
||||
// Whether the TF Lite GPU or CPU backend should be used. Effective only when
|
||||
// input tensors are on CPU. For input tensors on GPU, GPU backend is always
|
||||
// used.
|
||||
optional bool use_gpu = 2 [default = false];
|
||||
// DEPRECATED: configure "delegate" instead.
|
||||
optional bool use_gpu = 2 [deprecated = true, default = false];
|
||||
|
||||
// Android only. When true, an NNAPI delegate will be used for inference.
|
||||
// If NNAPI is not available, then the default CPU delegate will be used
|
||||
// automatically.
|
||||
optional bool use_nnapi = 3 [default = false];
|
||||
// DEPRECATED: configure "delegate" instead.
|
||||
optional bool use_nnapi = 3 [deprecated = true, default = false];
|
||||
|
||||
// The number of threads available to the interpreter. Effective only when
|
||||
// input tensors are on CPU and 'use_gpu' is false.
|
||||
optional int32 cpu_num_thread = 4 [default = -1];
|
||||
|
||||
// TfLite delegate to run inference.
|
||||
// NOTE: calculator is free to choose delegate if not specified explicitly.
|
||||
// NOTE: use_gpu/use_nnapi are ignored if specified. (Delegate takes
|
||||
// precedence over use_* deprecated options.)
|
||||
optional Delegate delegate = 5;
|
||||
}
|
||||
|
||||
@@ -16,6 +16,8 @@
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "absl/strings/str_replace.h"
|
||||
#include "absl/strings/string_view.h"
|
||||
#include "mediapipe/calculators/tflite/tflite_inference_calculator.pb.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/calculator_runner.h"
|
||||
@@ -39,13 +41,7 @@ namespace mediapipe {
|
||||
|
||||
using ::tflite::Interpreter;
|
||||
|
||||
class TfLiteInferenceCalculatorTest : public ::testing::Test {
|
||||
protected:
|
||||
std::unique_ptr<CalculatorRunner> runner_ = nullptr;
|
||||
};
|
||||
|
||||
// Tests a simple add model that adds an input tensor to itself.
|
||||
TEST_F(TfLiteInferenceCalculatorTest, SmokeTest) {
|
||||
void DoSmokeTest(absl::string_view delegate) {
|
||||
const int width = 8;
|
||||
const int height = 8;
|
||||
const int channels = 3;
|
||||
@@ -73,23 +69,24 @@ TEST_F(TfLiteInferenceCalculatorTest, SmokeTest) {
|
||||
auto input_vec = absl::make_unique<std::vector<TfLiteTensor>>();
|
||||
input_vec->emplace_back(*tensor);
|
||||
|
||||
std::string graph_proto = R"(
|
||||
input_stream: "tensor_in"
|
||||
node {
|
||||
calculator: "TfLiteInferenceCalculator"
|
||||
input_stream: "TENSORS:tensor_in"
|
||||
output_stream: "TENSORS:tensor_out"
|
||||
options {
|
||||
[mediapipe.TfLiteInferenceCalculatorOptions.ext] {
|
||||
model_path: "mediapipe/calculators/tflite/testdata/add.bin"
|
||||
$delegate
|
||||
}
|
||||
}
|
||||
}
|
||||
)";
|
||||
ASSERT_EQ(absl::StrReplaceAll({{"$delegate", delegate}}, &graph_proto), 1);
|
||||
// Prepare single calculator graph to and wait for packets.
|
||||
CalculatorGraphConfig graph_config =
|
||||
::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"(
|
||||
input_stream: "tensor_in"
|
||||
node {
|
||||
calculator: "TfLiteInferenceCalculator"
|
||||
input_stream: "TENSORS:tensor_in"
|
||||
output_stream: "TENSORS:tensor_out"
|
||||
options {
|
||||
[mediapipe.TfLiteInferenceCalculatorOptions.ext] {
|
||||
use_gpu: false
|
||||
model_path: "mediapipe/calculators/tflite/testdata/add.bin"
|
||||
}
|
||||
}
|
||||
}
|
||||
)");
|
||||
::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(graph_proto);
|
||||
std::vector<Packet> output_packets;
|
||||
tool::AddVectorSink("tensor_out", &graph_config, &output_packets);
|
||||
CalculatorGraph graph(graph_config);
|
||||
@@ -120,4 +117,10 @@ TEST_F(TfLiteInferenceCalculatorTest, SmokeTest) {
|
||||
MP_ASSERT_OK(graph.WaitUntilDone());
|
||||
}
|
||||
|
||||
// Tests a simple add model that adds an input tensor to itself.
|
||||
TEST(TfLiteInferenceCalculatorTest, SmokeTest) {
|
||||
DoSmokeTest(/*delegate=*/"");
|
||||
DoSmokeTest(/*delegate=*/"delegate { tflite {} }");
|
||||
}
|
||||
|
||||
} // namespace mediapipe
|
||||
|
||||
@@ -28,6 +28,21 @@ namespace mediapipe {
|
||||
// TENSORS - Vector of TfLiteTensor of type kTfLiteFloat32. Only the first
|
||||
// tensor will be used. The size of the values must be
|
||||
// (num_dimension x num_landmarks).
|
||||
//
|
||||
// FLIP_HORIZONTALLY (optional): Whether to flip landmarks horizontally or
|
||||
// not. Overrides corresponding side packet and/or field in the calculator
|
||||
// options.
|
||||
//
|
||||
// FLIP_VERTICALLY (optional): Whether to flip landmarks vertically or not.
|
||||
// Overrides corresponding side packet and/or field in the calculator options.
|
||||
//
|
||||
// Input side packet:
|
||||
// FLIP_HORIZONTALLY (optional): Whether to flip landmarks horizontally or
|
||||
// not. Overrides the corresponding field in the calculator options.
|
||||
//
|
||||
// FLIP_VERTICALLY (optional): Whether to flip landmarks vertically or not.
|
||||
// Overrides the corresponding field in the calculator options.
|
||||
//
|
||||
// Output:
|
||||
// LANDMARKS(optional) - Result MediaPipe landmarks.
|
||||
// NORM_LANDMARKS(optional) - Result MediaPipe normalized landmarks.
|
||||
@@ -61,6 +76,8 @@ class TfLiteTensorsToLandmarksCalculator : public CalculatorBase {
|
||||
private:
|
||||
::mediapipe::Status LoadOptions(CalculatorContext* cc);
|
||||
int num_landmarks_ = 0;
|
||||
bool flip_vertically_ = false;
|
||||
bool flip_horizontally_ = false;
|
||||
|
||||
::mediapipe::TfLiteTensorsToLandmarksCalculatorOptions options_;
|
||||
};
|
||||
@@ -75,6 +92,22 @@ REGISTER_CALCULATOR(TfLiteTensorsToLandmarksCalculator);
|
||||
cc->Inputs().Tag("TENSORS").Set<std::vector<TfLiteTensor>>();
|
||||
}
|
||||
|
||||
if (cc->Inputs().HasTag("FLIP_HORIZONTALLY")) {
|
||||
cc->Inputs().Tag("FLIP_HORIZONTALLY").Set<bool>();
|
||||
}
|
||||
|
||||
if (cc->Inputs().HasTag("FLIP_VERTICALLY")) {
|
||||
cc->Inputs().Tag("FLIP_VERTICALLY").Set<bool>();
|
||||
}
|
||||
|
||||
if (cc->InputSidePackets().HasTag("FLIP_HORIZONTALLY")) {
|
||||
cc->InputSidePackets().Tag("FLIP_HORIZONTALLY").Set<bool>();
|
||||
}
|
||||
|
||||
if (cc->InputSidePackets().HasTag("FLIP_VERTICALLY")) {
|
||||
cc->InputSidePackets().Tag("FLIP_VERTICALLY").Set<bool>();
|
||||
}
|
||||
|
||||
if (cc->Outputs().HasTag("LANDMARKS")) {
|
||||
cc->Outputs().Tag("LANDMARKS").Set<LandmarkList>();
|
||||
}
|
||||
@@ -98,17 +131,40 @@ REGISTER_CALCULATOR(TfLiteTensorsToLandmarksCalculator);
|
||||
<< "Must provide input with/height for getting normalized landmarks.";
|
||||
}
|
||||
if (cc->Outputs().HasTag("LANDMARKS") &&
|
||||
(options_.flip_vertically() || options_.flip_horizontally())) {
|
||||
(options_.flip_vertically() || options_.flip_horizontally() ||
|
||||
cc->InputSidePackets().HasTag("FLIP_HORIZONTALLY") ||
|
||||
cc->InputSidePackets().HasTag("FLIP_VERTICALLY"))) {
|
||||
RET_CHECK(options_.has_input_image_height() &&
|
||||
options_.has_input_image_width())
|
||||
<< "Must provide input with/height for using flip_vertically option "
|
||||
"when outputing landmarks in absolute coordinates.";
|
||||
}
|
||||
|
||||
flip_horizontally_ =
|
||||
cc->InputSidePackets().HasTag("FLIP_HORIZONTALLY")
|
||||
? cc->InputSidePackets().Tag("FLIP_HORIZONTALLY").Get<bool>()
|
||||
: options_.flip_horizontally();
|
||||
|
||||
flip_horizontally_ =
|
||||
cc->InputSidePackets().HasTag("FLIP_VERTICALLY")
|
||||
? cc->InputSidePackets().Tag("FLIP_VERTICALLY").Get<bool>()
|
||||
: options_.flip_vertically();
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status TfLiteTensorsToLandmarksCalculator::Process(
|
||||
CalculatorContext* cc) {
|
||||
// Override values if specified so.
|
||||
if (cc->Inputs().HasTag("FLIP_HORIZONTALLY") &&
|
||||
!cc->Inputs().Tag("FLIP_HORIZONTALLY").IsEmpty()) {
|
||||
flip_horizontally_ = cc->Inputs().Tag("FLIP_HORIZONTALLY").Get<bool>();
|
||||
}
|
||||
if (cc->Inputs().HasTag("FLIP_VERTICALLY") &&
|
||||
!cc->Inputs().Tag("FLIP_VERTICALLY").IsEmpty()) {
|
||||
flip_vertically_ = cc->Inputs().Tag("FLIP_VERTICALLY").Get<bool>();
|
||||
}
|
||||
|
||||
if (cc->Inputs().Tag("TENSORS").IsEmpty()) {
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
@@ -133,13 +189,13 @@ REGISTER_CALCULATOR(TfLiteTensorsToLandmarksCalculator);
|
||||
const int offset = ld * num_dimensions;
|
||||
Landmark* landmark = output_landmarks.add_landmark();
|
||||
|
||||
if (options_.flip_horizontally()) {
|
||||
if (flip_horizontally_) {
|
||||
landmark->set_x(options_.input_image_width() - raw_landmarks[offset]);
|
||||
} else {
|
||||
landmark->set_x(raw_landmarks[offset]);
|
||||
}
|
||||
if (num_dimensions > 1) {
|
||||
if (options_.flip_vertically()) {
|
||||
if (flip_vertically_) {
|
||||
landmark->set_y(options_.input_image_height() -
|
||||
raw_landmarks[offset + 1]);
|
||||
} else {
|
||||
|
||||
@@ -39,6 +39,15 @@ proto_library(
|
||||
],
|
||||
)
|
||||
|
||||
proto_library(
|
||||
name = "timed_box_list_id_to_label_calculator_proto",
|
||||
srcs = ["timed_box_list_id_to_label_calculator.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_proto",
|
||||
],
|
||||
)
|
||||
|
||||
proto_library(
|
||||
name = "latency_proto",
|
||||
srcs = ["latency.proto"],
|
||||
@@ -113,6 +122,18 @@ mediapipe_cc_proto_library(
|
||||
],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "timed_box_list_id_to_label_calculator_cc_proto",
|
||||
srcs = ["timed_box_list_id_to_label_calculator.proto"],
|
||||
cc_deps = [
|
||||
"//mediapipe/framework:calculator_cc_proto",
|
||||
],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
":timed_box_list_id_to_label_calculator_proto",
|
||||
],
|
||||
)
|
||||
|
||||
mediapipe_cc_proto_library(
|
||||
name = "latency_cc_proto",
|
||||
srcs = ["latency.proto"],
|
||||
@@ -313,6 +334,34 @@ cc_library(
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "timed_box_list_id_to_label_calculator",
|
||||
srcs = ["timed_box_list_id_to_label_calculator.cc"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
":timed_box_list_id_to_label_calculator_cc_proto",
|
||||
"//mediapipe/framework/port:status",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:packet",
|
||||
"//mediapipe/util/tracking:box_tracker_cc_proto",
|
||||
"//mediapipe/util:resource_util",
|
||||
] + select({
|
||||
"//mediapipe:android": [
|
||||
"//mediapipe/util/android/file/base",
|
||||
],
|
||||
"//mediapipe:apple": [
|
||||
"//mediapipe/util/android/file/base",
|
||||
],
|
||||
"//mediapipe:macos": [
|
||||
"//mediapipe/framework/port:file_helpers",
|
||||
],
|
||||
"//conditions:default": [
|
||||
"//mediapipe/framework/port:file_helpers",
|
||||
],
|
||||
}),
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "non_max_suppression_calculator",
|
||||
srcs = ["non_max_suppression_calculator.cc"],
|
||||
@@ -437,6 +486,7 @@ cc_library(
|
||||
"//mediapipe/framework/formats:rect_cc_proto",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/port:status",
|
||||
"@com_google_absl//absl/types:optional",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
@@ -928,6 +978,19 @@ cc_library(
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "local_file_pattern_contents_calculator",
|
||||
srcs = ["local_file_pattern_contents_calculator.cc"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework/port:file_helpers",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/port:status",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "filter_collection_calculator",
|
||||
srcs = ["filter_collection_calculator.cc"],
|
||||
|
||||
@@ -39,13 +39,13 @@ namespace mediapipe {
|
||||
|
||||
namespace {
|
||||
|
||||
constexpr char kInputFrameTag[] = "INPUT_FRAME";
|
||||
constexpr char kOutputFrameTag[] = "OUTPUT_FRAME";
|
||||
constexpr char kInputFrameTag[] = "IMAGE";
|
||||
constexpr char kOutputFrameTag[] = "IMAGE";
|
||||
|
||||
constexpr char kInputVectorTag[] = "VECTOR";
|
||||
|
||||
constexpr char kInputFrameTagGpu[] = "INPUT_FRAME_GPU";
|
||||
constexpr char kOutputFrameTagGpu[] = "OUTPUT_FRAME_GPU";
|
||||
constexpr char kInputFrameTagGpu[] = "IMAGE_GPU";
|
||||
constexpr char kOutputFrameTagGpu[] = "IMAGE_GPU";
|
||||
|
||||
enum { ATTRIB_VERTEX, ATTRIB_TEXTURE_POSITION, NUM_ATTRIBUTES };
|
||||
|
||||
@@ -61,7 +61,7 @@ constexpr int kAnnotationBackgroundColor[] = {100, 101, 102};
|
||||
// A calculator for rendering data on images.
|
||||
//
|
||||
// Inputs:
|
||||
// 1. INPUT_FRAME or INPUT_FRAME_GPU (optional): An ImageFrame (or GpuBuffer)
|
||||
// 1. IMAGE or IMAGE_GPU (optional): An ImageFrame (or GpuBuffer)
|
||||
// containing the input image.
|
||||
// If output is CPU, and input isn't provided, the renderer creates a
|
||||
// blank canvas with the width, height and color provided in the options.
|
||||
@@ -73,7 +73,7 @@ constexpr int kAnnotationBackgroundColor[] = {100, 101, 102};
|
||||
// input vector items. These input streams are tagged with "VECTOR".
|
||||
//
|
||||
// Output:
|
||||
// 1. OUTPUT_FRAME or OUTPUT_FRAME_GPU: A rendered ImageFrame (or GpuBuffer).
|
||||
// 1. IMAGE or IMAGE_GPU: A rendered ImageFrame (or GpuBuffer).
|
||||
//
|
||||
// For CPU input frames, only SRGBA, SRGB and GRAY8 format are supported. The
|
||||
// output format is the same as input except for GRAY8 where the output is in
|
||||
@@ -87,13 +87,13 @@ constexpr int kAnnotationBackgroundColor[] = {100, 101, 102};
|
||||
// Example config (CPU):
|
||||
// node {
|
||||
// calculator: "AnnotationOverlayCalculator"
|
||||
// input_stream: "INPUT_FRAME:image_frames"
|
||||
// input_stream: "IMAGE:image_frames"
|
||||
// input_stream: "render_data_1"
|
||||
// input_stream: "render_data_2"
|
||||
// input_stream: "render_data_3"
|
||||
// input_stream: "VECTOR:0:render_data_vec_0"
|
||||
// input_stream: "VECTOR:1:render_data_vec_1"
|
||||
// output_stream: "OUTPUT_FRAME:decorated_frames"
|
||||
// output_stream: "IMAGE:decorated_frames"
|
||||
// options {
|
||||
// [mediapipe.AnnotationOverlayCalculatorOptions.ext] {
|
||||
// }
|
||||
@@ -103,13 +103,13 @@ constexpr int kAnnotationBackgroundColor[] = {100, 101, 102};
|
||||
// Example config (GPU):
|
||||
// node {
|
||||
// calculator: "AnnotationOverlayCalculator"
|
||||
// input_stream: "INPUT_FRAME_GPU:image_frames"
|
||||
// input_stream: "IMAGE_GPU:image_frames"
|
||||
// input_stream: "render_data_1"
|
||||
// input_stream: "render_data_2"
|
||||
// input_stream: "render_data_3"
|
||||
// input_stream: "VECTOR:0:render_data_vec_0"
|
||||
// input_stream: "VECTOR:1:render_data_vec_1"
|
||||
// output_stream: "OUTPUT_FRAME_GPU:decorated_frames"
|
||||
// output_stream: "IMAGE_GPU:decorated_frames"
|
||||
// options {
|
||||
// [mediapipe.AnnotationOverlayCalculatorOptions.ext] {
|
||||
// }
|
||||
|
||||
@@ -12,10 +12,10 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "mediapipe//framework/packet.h"
|
||||
#include "mediapipe/calculators/util/detection_label_id_to_text_calculator.pb.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/formats/detection.pb.h"
|
||||
#include "mediapipe/framework/packet.h"
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
#include "mediapipe/util/resource_util.h"
|
||||
|
||||
|
||||
@@ -39,7 +39,8 @@ constexpr char kNormRectsTag[] = "NORM_RECTS";
|
||||
} // namespace
|
||||
|
||||
::mediapipe::Status DetectionsToRectsCalculator::DetectionToRect(
|
||||
const Detection& detection, Rect* rect) {
|
||||
const Detection& detection, const DetectionSpec& detection_spec,
|
||||
Rect* rect) {
|
||||
const LocationData location_data = detection.location_data();
|
||||
RET_CHECK(location_data.format() == LocationData::BOUNDING_BOX)
|
||||
<< "Only Detection with formats of BOUNDING_BOX can be converted to Rect";
|
||||
@@ -52,7 +53,8 @@ constexpr char kNormRectsTag[] = "NORM_RECTS";
|
||||
}
|
||||
|
||||
::mediapipe::Status DetectionsToRectsCalculator::DetectionToNormalizedRect(
|
||||
const Detection& detection, NormalizedRect* rect) {
|
||||
const Detection& detection, const DetectionSpec& detection_spec,
|
||||
NormalizedRect* rect) {
|
||||
const LocationData location_data = detection.location_data();
|
||||
RET_CHECK(location_data.format() == LocationData::RELATIVE_BOUNDING_BOX)
|
||||
<< "Only Detection with formats of RELATIVE_BOUNDING_BOX can be "
|
||||
@@ -174,27 +176,31 @@ constexpr char kNormRectsTag[] = "NORM_RECTS";
|
||||
}
|
||||
}
|
||||
|
||||
std::pair<int, int> image_size;
|
||||
if (rotate_) {
|
||||
RET_CHECK(!cc->Inputs().Tag(kImageSizeTag).IsEmpty());
|
||||
image_size = cc->Inputs().Tag(kImageSizeTag).Get<std::pair<int, int>>();
|
||||
}
|
||||
// Get dynamic calculator options (e.g. `image_size`).
|
||||
const DetectionSpec detection_spec = GetDetectionSpec(cc);
|
||||
|
||||
if (cc->Outputs().HasTag(kRectTag)) {
|
||||
auto output_rect = absl::make_unique<Rect>();
|
||||
MP_RETURN_IF_ERROR(DetectionToRect(detections[0], output_rect.get()));
|
||||
MP_RETURN_IF_ERROR(
|
||||
DetectionToRect(detections[0], detection_spec, output_rect.get()));
|
||||
if (rotate_) {
|
||||
output_rect->set_rotation(ComputeRotation(detections[0], image_size));
|
||||
float rotation;
|
||||
MP_RETURN_IF_ERROR(
|
||||
ComputeRotation(detections[0], detection_spec, &rotation));
|
||||
output_rect->set_rotation(rotation);
|
||||
}
|
||||
cc->Outputs().Tag(kRectTag).Add(output_rect.release(),
|
||||
cc->InputTimestamp());
|
||||
}
|
||||
if (cc->Outputs().HasTag(kNormRectTag)) {
|
||||
auto output_rect = absl::make_unique<NormalizedRect>();
|
||||
MP_RETURN_IF_ERROR(
|
||||
DetectionToNormalizedRect(detections[0], output_rect.get()));
|
||||
MP_RETURN_IF_ERROR(DetectionToNormalizedRect(detections[0], detection_spec,
|
||||
output_rect.get()));
|
||||
if (rotate_) {
|
||||
output_rect->set_rotation(ComputeRotation(detections[0], image_size));
|
||||
float rotation;
|
||||
MP_RETURN_IF_ERROR(
|
||||
ComputeRotation(detections[0], detection_spec, &rotation));
|
||||
output_rect->set_rotation(rotation);
|
||||
}
|
||||
cc->Outputs()
|
||||
.Tag(kNormRectTag)
|
||||
@@ -203,11 +209,13 @@ constexpr char kNormRectsTag[] = "NORM_RECTS";
|
||||
if (cc->Outputs().HasTag(kRectsTag)) {
|
||||
auto output_rects = absl::make_unique<std::vector<Rect>>(detections.size());
|
||||
for (int i = 0; i < detections.size(); ++i) {
|
||||
MP_RETURN_IF_ERROR(
|
||||
DetectionToRect(detections[i], &(output_rects->at(i))));
|
||||
MP_RETURN_IF_ERROR(DetectionToRect(detections[i], detection_spec,
|
||||
&(output_rects->at(i))));
|
||||
if (rotate_) {
|
||||
output_rects->at(i).set_rotation(
|
||||
ComputeRotation(detections[i], image_size));
|
||||
float rotation;
|
||||
MP_RETURN_IF_ERROR(
|
||||
ComputeRotation(detections[i], detection_spec, &rotation));
|
||||
output_rects->at(i).set_rotation(rotation);
|
||||
}
|
||||
}
|
||||
cc->Outputs().Tag(kRectsTag).Add(output_rects.release(),
|
||||
@@ -217,11 +225,13 @@ constexpr char kNormRectsTag[] = "NORM_RECTS";
|
||||
auto output_rects =
|
||||
absl::make_unique<std::vector<NormalizedRect>>(detections.size());
|
||||
for (int i = 0; i < detections.size(); ++i) {
|
||||
MP_RETURN_IF_ERROR(
|
||||
DetectionToNormalizedRect(detections[i], &(output_rects->at(i))));
|
||||
MP_RETURN_IF_ERROR(DetectionToNormalizedRect(
|
||||
detections[i], detection_spec, &(output_rects->at(i))));
|
||||
if (rotate_) {
|
||||
output_rects->at(i).set_rotation(
|
||||
ComputeRotation(detections[i], image_size));
|
||||
float rotation;
|
||||
MP_RETURN_IF_ERROR(
|
||||
ComputeRotation(detections[i], detection_spec, &rotation));
|
||||
output_rects->at(i).set_rotation(rotation);
|
||||
}
|
||||
}
|
||||
cc->Outputs()
|
||||
@@ -232,21 +242,35 @@ constexpr char kNormRectsTag[] = "NORM_RECTS";
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
float DetectionsToRectsCalculator::ComputeRotation(
|
||||
const Detection& detection, const std::pair<int, int> image_size) {
|
||||
::mediapipe::Status DetectionsToRectsCalculator::ComputeRotation(
|
||||
const Detection& detection, const DetectionSpec& detection_spec,
|
||||
float* rotation) {
|
||||
const auto& location_data = detection.location_data();
|
||||
const auto& image_size = detection_spec.image_size;
|
||||
RET_CHECK(image_size) << "Image size is required to calculate rotation";
|
||||
|
||||
const float x0 = location_data.relative_keypoints(start_keypoint_index_).x() *
|
||||
image_size.first;
|
||||
image_size->first;
|
||||
const float y0 = location_data.relative_keypoints(start_keypoint_index_).y() *
|
||||
image_size.second;
|
||||
image_size->second;
|
||||
const float x1 = location_data.relative_keypoints(end_keypoint_index_).x() *
|
||||
image_size.first;
|
||||
image_size->first;
|
||||
const float y1 = location_data.relative_keypoints(end_keypoint_index_).y() *
|
||||
image_size.second;
|
||||
image_size->second;
|
||||
|
||||
float rotation = target_angle_ - std::atan2(-(y1 - y0), x1 - x0);
|
||||
*rotation = NormalizeRadians(target_angle_ - std::atan2(-(y1 - y0), x1 - x0));
|
||||
|
||||
return NormalizeRadians(rotation);
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
DetectionSpec DetectionsToRectsCalculator::GetDetectionSpec(
|
||||
const CalculatorContext* cc) {
|
||||
absl::optional<std::pair<int, int>> image_size;
|
||||
if (cc->Inputs().HasTag(kImageSizeTag)) {
|
||||
image_size = cc->Inputs().Tag(kImageSizeTag).Get<std::pair<int, int>>();
|
||||
}
|
||||
|
||||
return {image_size};
|
||||
}
|
||||
|
||||
REGISTER_CALCULATOR(DetectionsToRectsCalculator);
|
||||
|
||||
@@ -16,6 +16,7 @@
|
||||
|
||||
#include <cmath>
|
||||
|
||||
#include "absl/types/optional.h"
|
||||
#include "mediapipe/calculators/util/detections_to_rects_calculator.pb.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/calculator_options.pb.h"
|
||||
@@ -27,6 +28,13 @@
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
// Dynamic options passed as calculator `input_stream` that can be used for
|
||||
// calculation of rectangle or rotation for given detection. Does not include
|
||||
// static calculator options which are available via private fields.
|
||||
struct DetectionSpec {
|
||||
absl::optional<std::pair<int, int>> image_size;
|
||||
};
|
||||
|
||||
// A calculator that converts Detection proto to Rect proto.
|
||||
//
|
||||
// Detection is the format for encoding one or more detections in an image.
|
||||
@@ -81,13 +89,16 @@ class DetectionsToRectsCalculator : public CalculatorBase {
|
||||
::mediapipe::Status Process(CalculatorContext* cc) override;
|
||||
|
||||
protected:
|
||||
virtual float ComputeRotation(const ::mediapipe::Detection& detection,
|
||||
const std::pair<int, int> image_size);
|
||||
virtual ::mediapipe::Status DetectionToRect(
|
||||
const ::mediapipe::Detection& detection, ::mediapipe::Rect* rect);
|
||||
const ::mediapipe::Detection& detection,
|
||||
const DetectionSpec& detection_spec, ::mediapipe::Rect* rect);
|
||||
virtual ::mediapipe::Status DetectionToNormalizedRect(
|
||||
const ::mediapipe::Detection& detection,
|
||||
::mediapipe::NormalizedRect* rect);
|
||||
const DetectionSpec& detection_spec, ::mediapipe::NormalizedRect* rect);
|
||||
virtual ::mediapipe::Status ComputeRotation(
|
||||
const ::mediapipe::Detection& detection,
|
||||
const DetectionSpec& detection_spec, float* rotation);
|
||||
virtual DetectionSpec GetDetectionSpec(const CalculatorContext* cc);
|
||||
|
||||
static inline float NormalizeRadians(float angle) {
|
||||
return angle - 2 * M_PI * std::floor((angle - (-M_PI)) / (2 * M_PI));
|
||||
|
||||
@@ -12,20 +12,6 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <cmath>
|
||||
#include <vector>
|
||||
|
||||
@@ -67,6 +53,15 @@ constexpr char kLetterboxPaddingTag[] = "LETTERBOX_PADDING";
|
||||
// input_stream: "LETTERBOX_PADDING:letterbox_padding"
|
||||
// output_stream: "LANDMARKS:adjusted_landmarks"
|
||||
// }
|
||||
//
|
||||
// node {
|
||||
// calculator: "LandmarkLetterboxRemovalCalculator"
|
||||
// input_stream: "LANDMARKS:0:landmarks_0"
|
||||
// input_stream: "LANDMARKS:1:landmarks_1"
|
||||
// input_stream: "LETTERBOX_PADDING:letterbox_padding"
|
||||
// output_stream: "LANDMARKS:0:adjusted_landmarks_0"
|
||||
// output_stream: "LANDMARKS:1:adjusted_landmarks_1"
|
||||
// }
|
||||
class LandmarkLetterboxRemovalCalculator : public CalculatorBase {
|
||||
public:
|
||||
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
|
||||
@@ -74,10 +69,20 @@ class LandmarkLetterboxRemovalCalculator : public CalculatorBase {
|
||||
cc->Inputs().HasTag(kLetterboxPaddingTag))
|
||||
<< "Missing one or more input streams.";
|
||||
|
||||
cc->Inputs().Tag(kLandmarksTag).Set<NormalizedLandmarkList>();
|
||||
RET_CHECK_EQ(cc->Inputs().NumEntries(kLandmarksTag),
|
||||
cc->Outputs().NumEntries(kLandmarksTag))
|
||||
<< "Same number of input and output landmarks is required.";
|
||||
|
||||
for (CollectionItemId id = cc->Inputs().BeginId(kLandmarksTag);
|
||||
id != cc->Inputs().EndId(kLandmarksTag); ++id) {
|
||||
cc->Inputs().Get(id).Set<NormalizedLandmarkList>();
|
||||
}
|
||||
cc->Inputs().Tag(kLetterboxPaddingTag).Set<std::array<float, 4>>();
|
||||
|
||||
cc->Outputs().Tag(kLandmarksTag).Set<NormalizedLandmarkList>();
|
||||
for (CollectionItemId id = cc->Outputs().BeginId(kLandmarksTag);
|
||||
id != cc->Outputs().EndId(kLandmarksTag); ++id) {
|
||||
cc->Outputs().Get(id).Set<NormalizedLandmarkList>();
|
||||
}
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
@@ -89,38 +94,45 @@ class LandmarkLetterboxRemovalCalculator : public CalculatorBase {
|
||||
}
|
||||
|
||||
::mediapipe::Status Process(CalculatorContext* cc) override {
|
||||
// Only process if there's input landmarks.
|
||||
if (cc->Inputs().Tag(kLandmarksTag).IsEmpty()) {
|
||||
if (cc->Inputs().Tag(kLetterboxPaddingTag).IsEmpty()) {
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
const NormalizedLandmarkList& input_landmarks =
|
||||
cc->Inputs().Tag(kLandmarksTag).Get<NormalizedLandmarkList>();
|
||||
const auto& letterbox_padding =
|
||||
cc->Inputs().Tag(kLetterboxPaddingTag).Get<std::array<float, 4>>();
|
||||
|
||||
const float left = letterbox_padding[0];
|
||||
const float top = letterbox_padding[1];
|
||||
const float left_and_right = letterbox_padding[0] + letterbox_padding[2];
|
||||
const float top_and_bottom = letterbox_padding[1] + letterbox_padding[3];
|
||||
|
||||
NormalizedLandmarkList output_landmarks;
|
||||
for (int i = 0; i < input_landmarks.landmark_size(); ++i) {
|
||||
const NormalizedLandmark& landmark = input_landmarks.landmark(i);
|
||||
NormalizedLandmark* new_landmark = output_landmarks.add_landmark();
|
||||
const float new_x = (landmark.x() - left) / (1.0f - left_and_right);
|
||||
const float new_y = (landmark.y() - top) / (1.0f - top_and_bottom);
|
||||
CollectionItemId input_id = cc->Inputs().BeginId(kLandmarksTag);
|
||||
CollectionItemId output_id = cc->Outputs().BeginId(kLandmarksTag);
|
||||
// Number of inputs and outpus is the same according to the contract.
|
||||
for (; input_id != cc->Inputs().EndId(kLandmarksTag);
|
||||
++input_id, ++output_id) {
|
||||
const auto& input_packet = cc->Inputs().Get(input_id);
|
||||
if (input_packet.IsEmpty()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
new_landmark->set_x(new_x);
|
||||
new_landmark->set_y(new_y);
|
||||
// Keep z-coord as is.
|
||||
new_landmark->set_z(landmark.z());
|
||||
const NormalizedLandmarkList& input_landmarks =
|
||||
input_packet.Get<NormalizedLandmarkList>();
|
||||
NormalizedLandmarkList output_landmarks;
|
||||
for (int i = 0; i < input_landmarks.landmark_size(); ++i) {
|
||||
const NormalizedLandmark& landmark = input_landmarks.landmark(i);
|
||||
NormalizedLandmark* new_landmark = output_landmarks.add_landmark();
|
||||
const float new_x = (landmark.x() - left) / (1.0f - left_and_right);
|
||||
const float new_y = (landmark.y() - top) / (1.0f - top_and_bottom);
|
||||
|
||||
new_landmark->set_x(new_x);
|
||||
new_landmark->set_y(new_y);
|
||||
// Keep z-coord as is.
|
||||
new_landmark->set_z(landmark.z());
|
||||
}
|
||||
|
||||
cc->Outputs().Get(output_id).AddPacket(
|
||||
MakePacket<NormalizedLandmarkList>(output_landmarks)
|
||||
.At(cc->InputTimestamp()));
|
||||
}
|
||||
|
||||
cc->Outputs()
|
||||
.Tag(kLandmarksTag)
|
||||
.AddPacket(MakePacket<NormalizedLandmarkList>(output_landmarks)
|
||||
.At(cc->InputTimestamp()));
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
};
|
||||
|
||||
@@ -12,20 +12,6 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <cmath>
|
||||
#include <vector>
|
||||
|
||||
@@ -63,6 +49,15 @@ constexpr char kRectTag[] = "NORM_RECT";
|
||||
// input_stream: "NORM_RECT:rect"
|
||||
// output_stream: "NORM_LANDMARKS:projected_landmarks"
|
||||
// }
|
||||
//
|
||||
// node {
|
||||
// calculator: "LandmarkProjectionCalculator"
|
||||
// input_stream: "NORM_LANDMARKS:0:landmarks_0"
|
||||
// input_stream: "NORM_LANDMARKS:1:landmarks_1"
|
||||
// input_stream: "NORM_RECT:rect"
|
||||
// output_stream: "NORM_LANDMARKS:0:projected_landmarks_0"
|
||||
// output_stream: "NORM_LANDMARKS:1:projected_landmarks_1"
|
||||
// }
|
||||
class LandmarkProjectionCalculator : public CalculatorBase {
|
||||
public:
|
||||
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
|
||||
@@ -70,10 +65,20 @@ class LandmarkProjectionCalculator : public CalculatorBase {
|
||||
cc->Inputs().HasTag(kRectTag))
|
||||
<< "Missing one or more input streams.";
|
||||
|
||||
cc->Inputs().Tag(kLandmarksTag).Set<NormalizedLandmarkList>();
|
||||
RET_CHECK_EQ(cc->Inputs().NumEntries(kLandmarksTag),
|
||||
cc->Outputs().NumEntries(kLandmarksTag))
|
||||
<< "Same number of input and output landmarks is required.";
|
||||
|
||||
for (CollectionItemId id = cc->Inputs().BeginId(kLandmarksTag);
|
||||
id != cc->Inputs().EndId(kLandmarksTag); ++id) {
|
||||
cc->Inputs().Get(id).Set<NormalizedLandmarkList>();
|
||||
}
|
||||
cc->Inputs().Tag(kRectTag).Set<NormalizedRect>();
|
||||
|
||||
cc->Outputs().Tag(kLandmarksTag).Set<NormalizedLandmarkList>();
|
||||
for (CollectionItemId id = cc->Outputs().BeginId(kLandmarksTag);
|
||||
id != cc->Outputs().EndId(kLandmarksTag); ++id) {
|
||||
cc->Outputs().Get(id).Set<NormalizedLandmarkList>();
|
||||
}
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
@@ -85,41 +90,50 @@ class LandmarkProjectionCalculator : public CalculatorBase {
|
||||
}
|
||||
|
||||
::mediapipe::Status Process(CalculatorContext* cc) override {
|
||||
const auto& options =
|
||||
cc->Options<::mediapipe::LandmarkProjectionCalculatorOptions>();
|
||||
// Only process if there's input landmarks.
|
||||
if (cc->Inputs().Tag(kLandmarksTag).IsEmpty()) {
|
||||
if (cc->Inputs().Tag(kRectTag).IsEmpty()) {
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
const NormalizedLandmarkList& input_landmarks =
|
||||
cc->Inputs().Tag(kLandmarksTag).Get<NormalizedLandmarkList>();
|
||||
const auto& input_rect = cc->Inputs().Tag(kRectTag).Get<NormalizedRect>();
|
||||
|
||||
NormalizedLandmarkList output_landmarks;
|
||||
for (int i = 0; i < input_landmarks.landmark_size(); ++i) {
|
||||
const NormalizedLandmark& landmark = input_landmarks.landmark(i);
|
||||
NormalizedLandmark* new_landmark = output_landmarks.add_landmark();
|
||||
const auto& options =
|
||||
cc->Options<::mediapipe::LandmarkProjectionCalculatorOptions>();
|
||||
|
||||
const float x = landmark.x() - 0.5f;
|
||||
const float y = landmark.y() - 0.5f;
|
||||
const float angle = options.ignore_rotation() ? 0 : input_rect.rotation();
|
||||
float new_x = std::cos(angle) * x - std::sin(angle) * y;
|
||||
float new_y = std::sin(angle) * x + std::cos(angle) * y;
|
||||
CollectionItemId input_id = cc->Inputs().BeginId(kLandmarksTag);
|
||||
CollectionItemId output_id = cc->Outputs().BeginId(kLandmarksTag);
|
||||
// Number of inputs and outpus is the same according to the contract.
|
||||
for (; input_id != cc->Inputs().EndId(kLandmarksTag);
|
||||
++input_id, ++output_id) {
|
||||
const auto& input_packet = cc->Inputs().Get(input_id);
|
||||
if (input_packet.IsEmpty()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
new_x = new_x * input_rect.width() + input_rect.x_center();
|
||||
new_y = new_y * input_rect.height() + input_rect.y_center();
|
||||
const auto& input_landmarks = input_packet.Get<NormalizedLandmarkList>();
|
||||
NormalizedLandmarkList output_landmarks;
|
||||
for (int i = 0; i < input_landmarks.landmark_size(); ++i) {
|
||||
const NormalizedLandmark& landmark = input_landmarks.landmark(i);
|
||||
NormalizedLandmark* new_landmark = output_landmarks.add_landmark();
|
||||
|
||||
new_landmark->set_x(new_x);
|
||||
new_landmark->set_y(new_y);
|
||||
// Keep z-coord as is.
|
||||
new_landmark->set_z(landmark.z());
|
||||
const float x = landmark.x() - 0.5f;
|
||||
const float y = landmark.y() - 0.5f;
|
||||
const float angle =
|
||||
options.ignore_rotation() ? 0 : input_rect.rotation();
|
||||
float new_x = std::cos(angle) * x - std::sin(angle) * y;
|
||||
float new_y = std::sin(angle) * x + std::cos(angle) * y;
|
||||
|
||||
new_x = new_x * input_rect.width() + input_rect.x_center();
|
||||
new_y = new_y * input_rect.height() + input_rect.y_center();
|
||||
|
||||
new_landmark->set_x(new_x);
|
||||
new_landmark->set_y(new_y);
|
||||
// Keep z-coord as is.
|
||||
new_landmark->set_z(landmark.z());
|
||||
}
|
||||
|
||||
cc->Outputs().Get(output_id).AddPacket(
|
||||
MakePacket<NormalizedLandmarkList>(output_landmarks)
|
||||
.At(cc->InputTimestamp()));
|
||||
}
|
||||
|
||||
cc->Outputs()
|
||||
.Tag(kLandmarksTag)
|
||||
.AddPacket(MakePacket<NormalizedLandmarkList>(output_landmarks)
|
||||
.At(cc->InputTimestamp()));
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
};
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <memory>
|
||||
#include <string>
|
||||
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/port/file_helpers.h"
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
|
||||
namespace mediapipe {
|
||||
// The calculator takes the path to local directory and desired file suffix to
|
||||
// mach as input side packets, and outputs the contents of those files that
|
||||
// match the pattern. Those matched files will be sent sequentially through the
|
||||
// output stream with incremental timestamp difference by 1.
|
||||
//
|
||||
// Example config:
|
||||
// node {
|
||||
// calculator: "LocalFilePatternContentsCalculator"
|
||||
// input_side_packet: "FILE_DIRECTORY:file_directory"
|
||||
// input_side_packet: "FILE_SUFFIX:file_suffix"
|
||||
// output_stream: "CONTENTS:contents"
|
||||
// }
|
||||
class LocalFilePatternContentsCalculator : public CalculatorBase {
|
||||
public:
|
||||
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
|
||||
cc->InputSidePackets().Tag("FILE_DIRECTORY").Set<std::string>();
|
||||
cc->InputSidePackets().Tag("FILE_SUFFIX").Set<std::string>();
|
||||
cc->Outputs().Tag("CONTENTS").Set<std::string>();
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status Open(CalculatorContext* cc) override {
|
||||
MP_RETURN_IF_ERROR(::mediapipe::file::MatchFileTypeInDirectory(
|
||||
cc->InputSidePackets().Tag("FILE_DIRECTORY").Get<std::string>(),
|
||||
cc->InputSidePackets().Tag("FILE_SUFFIX").Get<std::string>(),
|
||||
&filenames_));
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status Process(CalculatorContext* cc) override {
|
||||
if (current_output_ < filenames_.size()) {
|
||||
auto contents = absl::make_unique<std::string>();
|
||||
LOG(INFO) << filenames_[current_output_];
|
||||
MP_RETURN_IF_ERROR(mediapipe::file::GetContents(
|
||||
filenames_[current_output_], contents.get()));
|
||||
++current_output_;
|
||||
cc->Outputs()
|
||||
.Tag("CONTENTS")
|
||||
.Add(contents.release(), Timestamp(current_output_));
|
||||
} else {
|
||||
return tool::StatusStop();
|
||||
}
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
private:
|
||||
std::vector<std::string> filenames_;
|
||||
int current_output_ = 0;
|
||||
};
|
||||
|
||||
REGISTER_CALCULATOR(LocalFilePatternContentsCalculator);
|
||||
|
||||
} // namespace mediapipe
|
||||
@@ -45,15 +45,17 @@ RenderAnnotation::Rectangle* NewRect(
|
||||
void SetRect(bool normalized, double xmin, double ymin, double width,
|
||||
double height, double rotation,
|
||||
RenderAnnotation::Rectangle* rect) {
|
||||
if (xmin + width < 0.0 || ymin + height < 0.0) return;
|
||||
if (normalized) {
|
||||
if (xmin > 1.0 || ymin > 1.0) return;
|
||||
if (rotation == 0.0) {
|
||||
if (xmin + width < 0.0 || ymin + height < 0.0) return;
|
||||
if (normalized) {
|
||||
if (xmin > 1.0 || ymin > 1.0) return;
|
||||
}
|
||||
}
|
||||
rect->set_normalized(normalized);
|
||||
rect->set_left(normalized ? std::max(xmin, 0.0) : xmin);
|
||||
rect->set_top(normalized ? std::max(ymin, 0.0) : ymin);
|
||||
rect->set_right(normalized ? std::min(xmin + width, 1.0) : xmin + width);
|
||||
rect->set_bottom(normalized ? std::min(ymin + height, 1.0) : ymin + height);
|
||||
rect->set_left(xmin);
|
||||
rect->set_top(ymin);
|
||||
rect->set_right(xmin + width);
|
||||
rect->set_bottom(ymin + height);
|
||||
rect->set_rotation(rotation);
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,105 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "mediapipe/calculators/util/timed_box_list_id_to_label_calculator.pb.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/packet.h"
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
#include "mediapipe/util/resource_util.h"
|
||||
#include "mediapipe/util/tracking/box_tracker.pb.h"
|
||||
|
||||
#if defined(MEDIAPIPE_MOBILE)
|
||||
#include "mediapipe/util/android/file/base/file.h"
|
||||
#include "mediapipe/util/android/file/base/helpers.h"
|
||||
#else
|
||||
#include "mediapipe/framework/port/file_helpers.h"
|
||||
#endif
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
using mediapipe::TimedBoxProto;
|
||||
using mediapipe::TimedBoxProtoList;
|
||||
|
||||
// Takes a label map (from label IDs to names), and populate the label field in
|
||||
// TimedBoxProto according to it's ID.
|
||||
//
|
||||
// Example usage:
|
||||
// node {
|
||||
// calculator: "TimedBoxListIdToLabelCalculator"
|
||||
// input_stream: "input_timed_box_list"
|
||||
// output_stream: "output_timed_box_list"
|
||||
// node_options: {
|
||||
// [mediapipe.TimedBoxListIdToLabelCalculatorOptions] {
|
||||
// label_map_path: "labelmap.txt"
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
class TimedBoxListIdToLabelCalculator : public CalculatorBase {
|
||||
public:
|
||||
static ::mediapipe::Status GetContract(CalculatorContract* cc);
|
||||
|
||||
::mediapipe::Status Open(CalculatorContext* cc) override;
|
||||
::mediapipe::Status Process(CalculatorContext* cc) override;
|
||||
|
||||
private:
|
||||
std::unordered_map<int, std::string> label_map_;
|
||||
};
|
||||
REGISTER_CALCULATOR(TimedBoxListIdToLabelCalculator);
|
||||
|
||||
::mediapipe::Status TimedBoxListIdToLabelCalculator::GetContract(
|
||||
CalculatorContract* cc) {
|
||||
cc->Inputs().Index(0).Set<TimedBoxProtoList>();
|
||||
cc->Outputs().Index(0).Set<TimedBoxProtoList>();
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status TimedBoxListIdToLabelCalculator::Open(
|
||||
CalculatorContext* cc) {
|
||||
cc->SetOffset(TimestampDiff(0));
|
||||
|
||||
const auto& options =
|
||||
cc->Options<::mediapipe::TimedBoxListIdToLabelCalculatorOptions>();
|
||||
|
||||
std::string string_path;
|
||||
ASSIGN_OR_RETURN(string_path, PathToResourceAsFile(options.label_map_path()));
|
||||
std::string label_map_string;
|
||||
MP_RETURN_IF_ERROR(file::GetContents(string_path, &label_map_string));
|
||||
|
||||
std::istringstream stream(label_map_string);
|
||||
std::string line;
|
||||
int i = 0;
|
||||
while (std::getline(stream, line)) {
|
||||
label_map_[i++] = line;
|
||||
}
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status TimedBoxListIdToLabelCalculator::Process(
|
||||
CalculatorContext* cc) {
|
||||
const auto& input_list = cc->Inputs().Index(0).Get<TimedBoxProtoList>();
|
||||
auto output_list = absl::make_unique<TimedBoxProtoList>();
|
||||
for (const auto& input_box : input_list.box()) {
|
||||
TimedBoxProto* box_ptr = output_list->add_box();
|
||||
*box_ptr = input_box;
|
||||
|
||||
if (label_map_.find(input_box.id()) != label_map_.end()) {
|
||||
box_ptr->set_label(label_map_[input_box.id()]);
|
||||
}
|
||||
}
|
||||
cc->Outputs().Index(0).Add(output_list.release(), cc->InputTimestamp());
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
} // namespace mediapipe
|
||||
@@ -0,0 +1,28 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
syntax = "proto2";
|
||||
|
||||
package mediapipe;
|
||||
|
||||
import "mediapipe/framework/calculator.proto";
|
||||
|
||||
message TimedBoxListIdToLabelCalculatorOptions {
|
||||
extend mediapipe.CalculatorOptions {
|
||||
optional TimedBoxListIdToLabelCalculatorOptions ext = 297701606;
|
||||
}
|
||||
|
||||
// Path to a label map file for getting the actual name of detected classes.
|
||||
optional string label_map_path = 1;
|
||||
}
|
||||
@@ -66,6 +66,25 @@ void AddTimedBoxProtoToRenderData(
|
||||
rect->set_bottom(box_proto.bottom());
|
||||
rect->set_rotation(box_proto.rotation());
|
||||
}
|
||||
|
||||
if (box_proto.has_label()) {
|
||||
auto* label_annotation = render_data->add_render_annotations();
|
||||
label_annotation->mutable_color()->set_r(options.box_color().r());
|
||||
label_annotation->mutable_color()->set_g(options.box_color().g());
|
||||
label_annotation->mutable_color()->set_b(options.box_color().b());
|
||||
label_annotation->set_thickness(options.thickness());
|
||||
RenderAnnotation::Text* text = label_annotation->mutable_text();
|
||||
text->set_display_text(box_proto.label());
|
||||
text->set_normalized(true);
|
||||
constexpr float text_left_start = 0.3f;
|
||||
text->set_left((1.0f - text_left_start) * box_proto.left() +
|
||||
text_left_start * box_proto.right());
|
||||
constexpr float text_baseline = 0.6f;
|
||||
text->set_baseline(text_baseline * box_proto.bottom() +
|
||||
(1.0f - text_baseline) * box_proto.top());
|
||||
constexpr float text_height = 0.2f;
|
||||
text->set_font_height((box_proto.bottom() - box_proto.top()) * text_height);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
@@ -368,6 +368,7 @@ cc_test(
|
||||
cc_test(
|
||||
name = "tvl1_optical_flow_calculator_test",
|
||||
srcs = ["tvl1_optical_flow_calculator_test.cc"],
|
||||
linkstatic = 1,
|
||||
deps = [
|
||||
":tvl1_optical_flow_calculator",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
|
||||
@@ -15,6 +15,9 @@ For overall context on AutoFlip, please read this
|
||||
|
||||
Run the following command to build the AutoFlip pipeline:
|
||||
|
||||
Note: AutoFlip currently only works with OpenCV 3 . Please verify your OpenCV
|
||||
version beforehand.
|
||||
|
||||
```bash
|
||||
bazel build -c opt --define MEDIAPIPE_DISABLE_GPU=1 mediapipe/examples/desktop/autoflip:run_autoflip
|
||||
```
|
||||
|
||||
@@ -44,6 +44,14 @@ graphs can be easily adapted to run on CPU v.s. GPU.
|
||||
[Object Detection and Tracking with GPU](./object_tracking_mobile_gpu.md) illustrates how to
|
||||
use MediaPipe for object detection and tracking.
|
||||
|
||||
### Objectron: 3D Object Detection and Tracking with GPU
|
||||
|
||||
[MediaPipe Objectron is 3D Object Detection with GPU](./objectron_mobile_gpu.md)
|
||||
illustrates mobile real-time 3D object detection and tracking pipeline for every
|
||||
day objects like shoes and chairs
|
||||
|
||||
* [Android](./objectron_mobile_gpu.md)
|
||||
|
||||
### Face Detection with GPU
|
||||
|
||||
[Face Detection with GPU](./face_detection_mobile_gpu.md) illustrates how to use
|
||||
|
||||
@@ -259,9 +259,9 @@ node {
|
||||
# Draws annotations and overlays them on top of the input images.
|
||||
node {
|
||||
calculator: "AnnotationOverlayCalculator"
|
||||
input_stream: "INPUT_FRAME:throttled_input_video"
|
||||
input_stream: "IMAGE:throttled_input_video"
|
||||
input_stream: "render_data"
|
||||
output_stream: "OUTPUT_FRAME:output_video"
|
||||
output_stream: "IMAGE:output_video"
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
@@ -229,9 +229,9 @@ node {
|
||||
# Draws annotations and overlays them on top of the input images.
|
||||
node {
|
||||
calculator: "AnnotationOverlayCalculator"
|
||||
input_stream: "INPUT_FRAME:input_video_cpu"
|
||||
input_stream: "IMAGE:input_video_cpu"
|
||||
input_stream: "render_data"
|
||||
output_stream: "OUTPUT_FRAME:output_video_cpu"
|
||||
output_stream: "IMAGE:output_video_cpu"
|
||||
}
|
||||
|
||||
# Transfers the annotated image from CPU back to GPU memory, to be sent out of
|
||||
|
||||
@@ -221,8 +221,8 @@ node {
|
||||
# Draws annotations and overlays them on top of the input images.
|
||||
node {
|
||||
calculator: "AnnotationOverlayCalculator"
|
||||
input_stream: "INPUT_FRAME_GPU:throttled_input_video"
|
||||
input_stream: "IMAGE_GPU:throttled_input_video"
|
||||
input_stream: "render_data"
|
||||
output_stream: "OUTPUT_FRAME_GPU:output_video"
|
||||
output_stream: "IMAGE_GPU:output_video"
|
||||
}
|
||||
```
|
||||
|
||||
@@ -136,10 +136,10 @@ node {
|
||||
# Draws annotations and overlays them on top of the input images.
|
||||
node {
|
||||
calculator: "AnnotationOverlayCalculator"
|
||||
input_stream: "INPUT_FRAME_GPU:throttled_input_video"
|
||||
input_stream: "IMAGE_GPU:throttled_input_video"
|
||||
input_stream: "detection_render_data"
|
||||
input_stream: "rect_render_data"
|
||||
output_stream: "OUTPUT_FRAME_GPU:output_video"
|
||||
output_stream: "IMAGE_GPU:output_video"
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
@@ -716,10 +716,10 @@ node {
|
||||
# Draws annotations and overlays them on top of the input images.
|
||||
node {
|
||||
calculator: "AnnotationOverlayCalculator"
|
||||
input_stream: "INPUT_FRAME_GPU:input_image"
|
||||
input_stream: "IMAGE_GPU:input_image"
|
||||
input_stream: "detection_render_data"
|
||||
input_stream: "landmark_render_data"
|
||||
input_stream: "rect_render_data"
|
||||
output_stream: "OUTPUT_FRAME_GPU:output_image"
|
||||
output_stream: "IMAGE_GPU:output_image"
|
||||
}
|
||||
```
|
||||
|
||||
|
After Width: | Height: | Size: 100 KiB |
|
After Width: | Height: | Size: 2.5 MiB |
|
After Width: | Height: | Size: 64 KiB |
|
After Width: | Height: | Size: 2.8 MiB |
|
After Width: | Height: | Size: 113 KiB |
@@ -40,7 +40,7 @@ To build and run iOS apps:
|
||||
$ cd mediapipe
|
||||
```
|
||||
|
||||
2. Install Bazel (version between 0.24.1 and 1.2.1).
|
||||
2. Install Bazel (version between 1.0.0 and 1.2.1).
|
||||
|
||||
Follow the official
|
||||
[Bazel documentation](https://docs.bazel.build/versions/master/install-ubuntu.html)
|
||||
@@ -152,7 +152,7 @@ To build and run iOS apps:
|
||||
$ cd mediapipe
|
||||
```
|
||||
|
||||
2. Install Bazel (version between 0.24.1 and 1.2.1).
|
||||
2. Install Bazel (version between 1.0.0 and 1.2.1).
|
||||
|
||||
Follow the official
|
||||
[Bazel documentation](https://docs.bazel.build/versions/master/install-redhat.html)
|
||||
@@ -241,7 +241,7 @@ To build and run iOS apps:
|
||||
$ cd mediapipe
|
||||
```
|
||||
|
||||
3. Install Bazel (version between 0.24.1 and 1.1.0).
|
||||
3. Install Bazel (version between 1.0.0 and 1.1.0).
|
||||
|
||||
Option 1. Use package manager tool to install Bazel 1.1.0
|
||||
|
||||
@@ -364,8 +364,10 @@ To build and run iOS apps:
|
||||
|
||||
### Installing on Windows Subsystem for Linux (WSL)
|
||||
|
||||
Note: WSL has historically not provided access to USB cameras. Mediapipe can use
|
||||
a video file as input.
|
||||
Note: The pre-built OpenCV packages don't support cameras in WSL. Unless you
|
||||
[compile](https://funvision.blogspot.com/2019/12/opencv-web-camera-and-video-streams-in.html)
|
||||
OpenCV with FFMPEG and GStreamer in WSL, the live demos won't work with any
|
||||
cameras. Alternatively, you use a video file as input.
|
||||
|
||||
1. Follow the
|
||||
[instruction](https://docs.microsoft.com/en-us/windows/wsl/install-win10) to
|
||||
@@ -373,7 +375,7 @@ a video file as input.
|
||||
|
||||
2. Install Windows ADB and start the ADB server in Windows.
|
||||
|
||||
Note: Window’s and WSL’s adb versions must be the same version, e.g., if WSL
|
||||
Note: Windows' and WSL’s adb versions must be the same version, e.g., if WSL
|
||||
has ADB 1.0.39, you need to download the corresponding Windows ADB from
|
||||
[here](https://dl.google.com/android/repository/platform-tools_r26.0.1-windows.zip).
|
||||
|
||||
@@ -389,18 +391,18 @@ a video file as input.
|
||||
username@DESKTOP-TMVLBJ1:~$ sudo apt-get update && sudo apt-get install -y build-essential git python zip adb openjdk-8-jdk
|
||||
```
|
||||
|
||||
5. Install Bazel (version between 0.24.1 and 1.2.1).
|
||||
5. Install Bazel (version between 1.0.0 and 1.2.1).
|
||||
|
||||
```bash
|
||||
username@DESKTOP-TMVLBJ1:~$ curl -sLO --retry 5 --retry-max-time 10 \
|
||||
https://storage.googleapis.com/bazel/0.27.0/release/bazel-0.27.0-installer-linux-x86_64.sh && \
|
||||
sudo mkdir -p /usr/local/bazel/0.27.0 && \
|
||||
chmod 755 bazel-0.27.0-installer-linux-x86_64.sh && \
|
||||
sudo ./bazel-0.27.0-installer-linux-x86_64.sh --prefix=/usr/local/bazel/0.27.0 && \
|
||||
source /usr/local/bazel/0.27.0/lib/bazel/bin/bazel-complete.bash
|
||||
https://storage.googleapis.com/bazel/1.0.0/release/bazel-1.0.0-installer-linux-x86_64.sh && \
|
||||
sudo mkdir -p /usr/local/bazel/1.0.0 && \
|
||||
chmod 755 bazel-1.0.0-installer-linux-x86_64.sh && \
|
||||
sudo ./bazel-1.0.0-installer-linux-x86_64.sh --prefix=/usr/local/bazel/1.0.0 && \
|
||||
source /usr/local/bazel/1.0.0/lib/bazel/bin/bazel-complete.bash
|
||||
|
||||
username@DESKTOP-TMVLBJ1:~$ /usr/local/bazel/0.27.0/lib/bazel/bin/bazel version && \
|
||||
alias bazel='/usr/local/bazel/0.27.0/lib/bazel/bin/bazel'
|
||||
username@DESKTOP-TMVLBJ1:~$ /usr/local/bazel/1.0.0/lib/bazel/bin/bazel version && \
|
||||
alias bazel='/usr/local/bazel/1.0.0/lib/bazel/bin/bazel'
|
||||
```
|
||||
|
||||
6. Checkout MediaPipe repository.
|
||||
|
||||
@@ -745,11 +745,11 @@ node {
|
||||
# a vector of RenderData objects and draws each of them on the input frame.
|
||||
node {
|
||||
calculator: "AnnotationOverlayCalculator"
|
||||
input_stream: "INPUT_FRAME_GPU:input_image"
|
||||
input_stream: "IMAGE_GPU:input_image"
|
||||
input_stream: "detection_render_data"
|
||||
input_stream: "multi_hand_rects_render_data"
|
||||
input_stream: "multi_palm_rects_render_data"
|
||||
input_stream: "VECTOR:0:multi_hand_landmarks_render_data"
|
||||
output_stream: "OUTPUT_FRAME_GPU:output_image"
|
||||
output_stream: "IMAGE_GPU:output_image"
|
||||
}
|
||||
```
|
||||
|
||||
@@ -26,6 +26,7 @@ To build and run the TensorFlow example on desktop, run:
|
||||
$ bazel build -c opt \
|
||||
--define MEDIAPIPE_DISABLE_GPU=1 \
|
||||
--define no_aws_support=true \
|
||||
--linkopt=-s \
|
||||
mediapipe/examples/desktop/object_detection:object_detection_tensorflow
|
||||
|
||||
# It should print:
|
||||
@@ -163,9 +164,9 @@ node {
|
||||
# the graph.
|
||||
node {
|
||||
calculator: "AnnotationOverlayCalculator"
|
||||
input_stream: "INPUT_FRAME:input_video"
|
||||
input_stream: "IMAGE:input_video"
|
||||
input_stream: "render_data"
|
||||
output_stream: "OUTPUT_FRAME:output_video"
|
||||
output_stream: "IMAGE:output_video"
|
||||
}
|
||||
|
||||
# Encodes the annotated images into a video file, adopting properties specified
|
||||
@@ -396,9 +397,9 @@ node {
|
||||
# the graph.
|
||||
node {
|
||||
calculator: "AnnotationOverlayCalculator"
|
||||
input_stream: "INPUT_FRAME:input_video"
|
||||
input_stream: "IMAGE:input_video"
|
||||
input_stream: "render_data"
|
||||
output_stream: "OUTPUT_FRAME:output_video"
|
||||
output_stream: "IMAGE:output_video"
|
||||
}
|
||||
|
||||
# Encodes the annotated images into a video file, adopting properties specified
|
||||
|
||||
@@ -230,9 +230,9 @@ node {
|
||||
# Draws annotations and overlays them on top of the input images.
|
||||
node {
|
||||
calculator: "AnnotationOverlayCalculator"
|
||||
input_stream: "INPUT_FRAME:throttled_input_video_cpu"
|
||||
input_stream: "IMAGE:throttled_input_video_cpu"
|
||||
input_stream: "render_data"
|
||||
output_stream: "OUTPUT_FRAME:output_video_cpu"
|
||||
output_stream: "IMAGE:output_video_cpu"
|
||||
}
|
||||
|
||||
# Transfers the annotated image from CPU back to GPU memory, to be sent out of
|
||||
|
||||
@@ -212,8 +212,8 @@ node {
|
||||
# Draws annotations and overlays them on top of the input images.
|
||||
node {
|
||||
calculator: "AnnotationOverlayCalculator"
|
||||
input_stream: "INPUT_FRAME_GPU:throttled_input_video"
|
||||
input_stream: "IMAGE_GPU:throttled_input_video"
|
||||
input_stream: "render_data"
|
||||
output_stream: "OUTPUT_FRAME_GPU:output_video"
|
||||
output_stream: "IMAGE_GPU:output_video"
|
||||
}
|
||||
```
|
||||
|
||||
@@ -467,9 +467,9 @@ node {
|
||||
# Draws annotations and overlays them on top of the input images.
|
||||
node {
|
||||
calculator: "AnnotationOverlayCalculator"
|
||||
input_stream: "INPUT_FRAME_GPU:input_image"
|
||||
input_stream: "IMAGE_GPU:input_image"
|
||||
input_stream: "detections_render_data"
|
||||
output_stream: "OUTPUT_FRAME_GPU:output_image"
|
||||
output_stream: "IMAGE_GPU:output_image"
|
||||
}
|
||||
|
||||
```
|
||||
@@ -484,7 +484,8 @@ CPU.
|
||||
To build and run the app:
|
||||
|
||||
```bash
|
||||
bazel build -c opt mediapipe/examples/desktop/object_tracking:object_tracking_cpu
|
||||
bazel build -c opt mediapipe/examples/desktop/object_tracking:object_tracking_cpu \
|
||||
--define MEDIAPIPE_DISABLE_GPU=1
|
||||
|
||||
bazel-bin/mediapipe/examples/desktop/object_tracking/object_tracking_cpu \
|
||||
--calculator_graph_config_file=mediapipe/graphs/tracking/object_detection_tracking_desktop_live.pbtxt
|
||||
|
||||
@@ -0,0 +1,489 @@
|
||||
# MediaPipe Objectron (GPU)
|
||||
|
||||
This doc focuses on the
|
||||
[below example graph](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection_3d/shoe_classic_occlusion_tracking.pbtxt)
|
||||
that performs 3D object detection and tracking with TensorFlow Lite on GPU.
|
||||
|
||||
Objectron for shoes | Objectron for chairs
|
||||
:-----------------------------------------------------------------------------: | :------------------:
|
||||
 | 
|
||||
|
||||
For overall context on MediaPipe Objectron, please read the
|
||||
[Google AI Blog](https://mediapipe.page.link/objectron-aiblog). The Objectron's
|
||||
ML model (see also the [model card](https://mediapipe.page.link/objectron-mc))
|
||||
estimates a 3D bounding box for the detected object.
|
||||
|
||||
## Android
|
||||
|
||||
[Source](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d)
|
||||
|
||||
An arm64 build of Objectron for shoes can be
|
||||
[downloaded here](https://drive.google.com/open?id=1S0K4hbWt3o31FfQ4QU3Rz7IHrvOUMx1d),
|
||||
and for chairs can be
|
||||
[downloaded here](https://drive.google.com/open?id=1MM8K-13bXLCVS1EHQ-KgkVyEahEPrKej).
|
||||
|
||||
To build and install the Objectron for shoes:
|
||||
|
||||
```bash
|
||||
bazel build -c opt --config android_arm64 mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d:objectdetection3d
|
||||
```
|
||||
|
||||
Similarly to build and install the Objectron for chairs, add **--define
|
||||
chair=true** flag to build command.
|
||||
|
||||
```bash
|
||||
bazel build -c opt --define chair=true --config android_arm64 mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d:objectdetection3d
|
||||
```
|
||||
|
||||
Once the app is built, install in on Android device with:
|
||||
|
||||
```bash
|
||||
adb install bazel-bin/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/objectdetection3d.apk
|
||||
```
|
||||
|
||||
## Graph
|
||||
|
||||
The Objectron main graph internally utilizes the Objectron detection subgraph,
|
||||
and the Objectron tracking subgraph. To visualize the graph as shown above, copy
|
||||
the text specification of the graph below and paste it into
|
||||
[MediaPipe Visualizer](https://viz.mediapipe.dev/).
|
||||
|
||||
### Main Graph
|
||||
|
||||
This is the main graph for the shoe detector. This graph runs detection and
|
||||
tracking and renders the output to the display.
|
||||
|
||||

|
||||
|
||||
[Source pbtxt file](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection_3d/shoe_classic_occlusion_tracking.pbtxt)
|
||||
|
||||
```bash
|
||||
# MediaPipe object detection 3D with tracking graph.
|
||||
|
||||
# Images on GPU coming into and out of the graph.
|
||||
input_stream: "input_video"
|
||||
output_stream: "output_video"
|
||||
|
||||
# Creates a copy of the input_video stream. At the end of the graph, the
|
||||
# GlAnimationOverlayCalculator will consume the input_video texture and draws
|
||||
# on top of it.
|
||||
node: {
|
||||
calculator: "GlScalerCalculator"
|
||||
input_stream: "VIDEO:input_video"
|
||||
output_stream: "VIDEO:input_video_copy"
|
||||
}
|
||||
|
||||
# Resamples the images by specific frame rate. This calculator is used to
|
||||
# control the frequecy of subsequent calculators/subgraphs, e.g. less power
|
||||
# consumption for expensive process.
|
||||
node {
|
||||
calculator: "PacketResamplerCalculator"
|
||||
input_stream: "DATA:input_video_copy"
|
||||
output_stream: "DATA:sampled_input_video"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.PacketResamplerCalculatorOptions] {
|
||||
frame_rate: 5
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
node {
|
||||
calculator: "ObjectronDetectionSubgraphGpu"
|
||||
input_stream: "IMAGE_GPU:sampled_input_video"
|
||||
output_stream: "ANNOTATIONS:objects"
|
||||
}
|
||||
|
||||
node {
|
||||
calculator: "ObjectronTrackingSubgraphGpu"
|
||||
input_stream: "FRAME_ANNOTATION:objects"
|
||||
input_stream: "IMAGE_GPU:input_video_copy"
|
||||
output_stream: "LIFTED_FRAME_ANNOTATION:lifted_tracked_objects"
|
||||
}
|
||||
|
||||
# The rendering nodes:
|
||||
# We are rendering two meshes: 1) a 3D bounding box, which we overlay directly
|
||||
# on the texture, and 2) a shoe CAD model, which we use as an occlusion mask.
|
||||
# These models are designed using different tools, so we supply a transformation
|
||||
# to bring both of them to the Objectron's coordinate system.
|
||||
|
||||
# Creates a model matrices for the tracked object given the lifted 3D points.
|
||||
# This calculator does two things: 1) Estimates object's pose (orientation,
|
||||
# translation, and scale) from the 3D vertices, and
|
||||
# 2) bring the object from the objectron's coordinate system to the renderer
|
||||
# (OpenGL) coordinate system. Since the final goal is to render a mesh file on
|
||||
# top of the object, we also supply a transformation to bring the mesh to the
|
||||
# objectron's coordinate system, and rescale mesh to the unit size.
|
||||
node {
|
||||
calculator: "AnnotationsToModelMatricesCalculator"
|
||||
input_stream: "ANNOTATIONS:lifted_tracked_objects"
|
||||
output_stream: "MODEL_MATRICES:model_matrices"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.AnnotationsToModelMatricesCalculatorOptions] {
|
||||
# Re-scale the CAD model to the size of a unit box
|
||||
model_scale: 0.05
|
||||
model_scale: 0.05
|
||||
model_scale: 0.05
|
||||
# Bring the box CAD model to objectron's coordinate system. This
|
||||
# is equivalent of -pi/2 rotation along the y-axis (right-hand rule):
|
||||
# Eigen::AngleAxisf(-M_PI / 2., Eigen::Vector3f::UnitY())
|
||||
model_transformation: 0.0
|
||||
model_transformation: 0.0
|
||||
model_transformation: -1.0
|
||||
model_transformation: 0.0
|
||||
model_transformation: 0.0
|
||||
model_transformation: 1.0
|
||||
model_transformation: 0.0
|
||||
model_transformation: 0.0
|
||||
model_transformation: 1.0
|
||||
model_transformation: 0.0
|
||||
model_transformation: 0.0
|
||||
model_transformation: 0.0
|
||||
model_transformation: 0.0
|
||||
model_transformation: 0.0
|
||||
model_transformation: 0.0
|
||||
model_transformation: 1.0
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Compute the model matrices for the CAD model of the shoe, to be used as an
|
||||
# occlusion mask. The model will be rendered at the exact same location as the
|
||||
# bounding box.
|
||||
node {
|
||||
calculator: "AnnotationsToModelMatricesCalculator"
|
||||
input_stream: "ANNOTATIONS:lifted_tracked_objects"
|
||||
output_stream: "MODEL_MATRICES:mask_model_matrices"
|
||||
#input_side_packet: "MODEL_SCALE:model_scale"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.AnnotationsToModelMatricesCalculatorOptions] {
|
||||
# Re-scale the CAD model to the size of a unit box
|
||||
model_scale: 0.45
|
||||
model_scale: 0.25
|
||||
model_scale: 0.15
|
||||
# Bring the shoe CAD model to Deep Pursuit 3D's coordinate system. This
|
||||
# is equivalent of -pi/2 rotation along the x-axis (right-hand rule):
|
||||
# Eigen::AngleAxisf(-M_PI / 2., Eigen::Vector3f::UnitX())
|
||||
model_transformation: 1.0
|
||||
model_transformation: 0.0
|
||||
model_transformation: 0.0
|
||||
model_transformation: 0.0
|
||||
model_transformation: 0.0
|
||||
model_transformation: 0.0
|
||||
model_transformation: 1.0
|
||||
model_transformation: 0.0
|
||||
model_transformation: 0.0
|
||||
model_transformation: -1.0
|
||||
model_transformation: 0.0
|
||||
model_transformation: 0.0
|
||||
model_transformation: 0.0
|
||||
model_transformation: 0.0
|
||||
model_transformation: 0.0
|
||||
model_transformation: 1.0
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Render everything together. First we render the 3D bounding box animation,
|
||||
# then we render the occlusion mask.
|
||||
node: {
|
||||
calculator: "GlAnimationOverlayCalculator"
|
||||
input_stream: "VIDEO:input_video"
|
||||
input_stream: "MODEL_MATRICES:model_matrices"
|
||||
input_stream: "MASK_MODEL_MATRICES:mask_model_matrices"
|
||||
output_stream: "output_video"
|
||||
input_side_packet: "TEXTURE:box_texture"
|
||||
input_side_packet: "ANIMATION_ASSET:box_asset_name"
|
||||
input_side_packet: "MASK_TEXTURE:obj_texture"
|
||||
input_side_packet: "MASK_ASSET:obj_asset_name"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.GlAnimationOverlayCalculatorOptions] {
|
||||
# Output resolution is 480x640 with the aspect ratio of 0.75
|
||||
aspect_ratio: 0.75
|
||||
vertical_fov_degrees: 70.
|
||||
animation_speed_fps: 25
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
```
|
||||
|
||||
### Objectron Detection Subgraph
|
||||
|
||||
Objectron detection subgraph uses the *TfLiteInferenceCalculator* to run
|
||||
inference and decodes the output tensor to *FrameAnnotation* protobuf. The
|
||||
*FrameAnnotation* contains nine keypoints: the bounding box's center, as well as
|
||||
its eight vertices. The boxes will be passed to the Objectron tracking subgraph.
|
||||
|
||||

|
||||
|
||||
[Source pbtxt file](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection_3d/subgraphs/objectron_detection_gpu.pbtxt)
|
||||
|
||||
```bash
|
||||
# MediaPipe Objectron detection gpu subgraph
|
||||
|
||||
type: "ObjectronDetectionSubgraphGpu"
|
||||
|
||||
input_stream: "IMAGE_GPU:input_video"
|
||||
output_stream: "ANNOTATIONS:objects"
|
||||
|
||||
# Transforms the input image on GPU to a 480x640 image. To scale the input
|
||||
# image, the scale_mode option is set to FIT to preserve the aspect ratio,
|
||||
# resulting in potential letterboxing in the transformed image.
|
||||
node: {
|
||||
calculator: "ImageTransformationCalculator"
|
||||
input_stream: "IMAGE_GPU:input_video"
|
||||
output_stream: "IMAGE_GPU:transformed_input_video"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.ImageTransformationCalculatorOptions] {
|
||||
output_width: 480
|
||||
output_height: 640
|
||||
scale_mode: FIT
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Converts the transformed input image on GPU into an image tensor stored as a
|
||||
# TfLiteTensor.
|
||||
node {
|
||||
calculator: "TfLiteConverterCalculator"
|
||||
input_stream: "IMAGE_GPU:transformed_input_video"
|
||||
output_stream: "TENSORS_GPU:image_tensor"
|
||||
}
|
||||
|
||||
# Generates a single side packet containing a TensorFlow Lite op resolver that
|
||||
# supports custom ops needed by the model used in this graph.
|
||||
node {
|
||||
calculator: "TfLiteCustomOpResolverCalculator"
|
||||
output_side_packet: "opresolver"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.TfLiteCustomOpResolverCalculatorOptions] {
|
||||
use_gpu: true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Runs a TensorFlow Lite model on GPU that takes an image tensor and outputs a
|
||||
# vector of tensors representing, for instance, detection boxes/keypoints and
|
||||
# scores.
|
||||
node {
|
||||
calculator: "TfLiteInferenceCalculator"
|
||||
input_stream: "TENSORS_GPU:image_tensor"
|
||||
output_stream: "TENSORS:detection_tensors"
|
||||
input_side_packet: "CUSTOM_OP_RESOLVER:opresolver"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.TfLiteInferenceCalculatorOptions] {
|
||||
model_path: "object_detection_3d.tflite"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Decodes the model's output tensor (the heatmap and the distance fields) to 2D
|
||||
# keypoints. There are nine 2D keypoints: one center keypoint and eight vertices
|
||||
# for the 3D bounding box. The calculator parameters determine's the decoder's
|
||||
# sensitivity.
|
||||
node {
|
||||
calculator: "TfLiteTensorsToObjectsCalculator"
|
||||
input_stream: "TENSORS:detection_tensors"
|
||||
output_stream: "ANNOTATIONS:objects"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.TfLiteTensorsToObjectsCalculatorOptions] {
|
||||
num_classes: 1
|
||||
num_keypoints: 9
|
||||
decoder_config {
|
||||
heatmap_threshold: 0.6
|
||||
local_max_distance: 2
|
||||
offset_scale_coef: 1.0
|
||||
voting_radius: 2
|
||||
voting_allowance: 1
|
||||
voting_threshold: 0.2
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Object Tracking Subgraph
|
||||
|
||||
Object tracking subgraph uses a *BoxTracker* calculator which is a generic
|
||||
tracking library, also used in
|
||||
[Mediapipe's 2D Object Detection and Tracking](https://github.com/google/mediapipe/tree/master/mediapipe/g3doc/object_tracking_mobile_gpu.md).
|
||||
The tracking runs every frame and when a new detection is available, it
|
||||
consolidates the detection and tracking results. The tracker tracks the box with
|
||||
its 2D keypoints, so at the end we lift the 2D keypoints to 3D using EPnP
|
||||
algorithm in *Lift2DFrameAnnotationTo3D* Calculator.
|
||||
|
||||

|
||||
|
||||
[Source pbtxt file](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection_3d/subgraphs/objectron_tracking_gpu.pbtxt)
|
||||
|
||||
```bash
|
||||
# MediaPipe Objectron tracking gpu subgraph
|
||||
|
||||
type: "ObjectronTrackingSubgraphGpu"
|
||||
|
||||
input_stream: "FRAME_ANNOTATION:objects"
|
||||
input_stream: "IMAGE_GPU:input_video"
|
||||
output_stream: "LIFTED_FRAME_ANNOTATION:lifted_tracked_objects"
|
||||
|
||||
|
||||
# Converts the detected keypoints to Boxes, used by the tracking subgraph.
|
||||
node {
|
||||
calculator: "FrameAnnotationToTimedBoxListCalculator"
|
||||
input_stream: "FRAME_ANNOTATION:objects"
|
||||
output_stream: "BOXES:start_pos"
|
||||
}
|
||||
|
||||
node: {
|
||||
calculator: "ImageTransformationCalculator"
|
||||
input_stream: "IMAGE_GPU:input_video"
|
||||
output_stream: "IMAGE_GPU:downscaled_input_video"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.ImageTransformationCalculatorOptions] {
|
||||
output_width: 240
|
||||
output_height: 320
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Converts GPU buffer to ImageFrame for processing tracking.
|
||||
node: {
|
||||
calculator: "GpuBufferToImageFrameCalculator"
|
||||
input_stream: "downscaled_input_video"
|
||||
output_stream: "downscaled_input_video_cpu"
|
||||
}
|
||||
|
||||
# Performs motion analysis on an incoming video stream.
|
||||
node: {
|
||||
calculator: "MotionAnalysisCalculator"
|
||||
input_stream: "VIDEO:downscaled_input_video_cpu"
|
||||
output_stream: "CAMERA:camera_motion"
|
||||
output_stream: "FLOW:region_flow"
|
||||
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.MotionAnalysisCalculatorOptions]: {
|
||||
analysis_options {
|
||||
analysis_policy: ANALYSIS_POLICY_CAMERA_MOBILE
|
||||
flow_options {
|
||||
fast_estimation_min_block_size: 100
|
||||
top_inlier_sets: 1
|
||||
frac_inlier_error_threshold: 3e-3
|
||||
downsample_mode: DOWNSAMPLE_TO_INPUT_SIZE
|
||||
verification_distance: 5.0
|
||||
verify_long_feature_acceleration: true
|
||||
verify_long_feature_trigger_ratio: 0.1
|
||||
tracking_options {
|
||||
max_features: 500
|
||||
adaptive_extraction_levels: 2
|
||||
min_eig_val_settings {
|
||||
adaptive_lowest_quality_level: 2e-4
|
||||
}
|
||||
klt_tracker_implementation: KLT_OPENCV
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Reads optical flow fields defined in
|
||||
# mediapipe/framework/formats/motion/optical_flow_field.h,
|
||||
# returns a VideoFrame with 2 channels (v_x and v_y), each channel is quantized
|
||||
# to 0-255.
|
||||
node: {
|
||||
calculator: "FlowPackagerCalculator"
|
||||
input_stream: "FLOW:region_flow"
|
||||
input_stream: "CAMERA:camera_motion"
|
||||
output_stream: "TRACKING:tracking_data"
|
||||
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.FlowPackagerCalculatorOptions]: {
|
||||
flow_packager_options: {
|
||||
binary_tracking_data_support: false
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Tracks box positions over time.
|
||||
node: {
|
||||
calculator: "BoxTrackerCalculator"
|
||||
input_stream: "TRACKING:tracking_data"
|
||||
input_stream: "TRACK_TIME:input_video"
|
||||
input_stream: "START_POS:start_pos"
|
||||
input_stream: "CANCEL_OBJECT_ID:cancel_object_id"
|
||||
input_stream_info: {
|
||||
tag_index: "CANCEL_OBJECT_ID"
|
||||
back_edge: true
|
||||
}
|
||||
output_stream: "BOXES:boxes"
|
||||
|
||||
input_stream_handler {
|
||||
input_stream_handler: "SyncSetInputStreamHandler"
|
||||
options {
|
||||
[mediapipe.SyncSetInputStreamHandlerOptions.ext] {
|
||||
sync_set {
|
||||
tag_index: "TRACKING"
|
||||
tag_index: "TRACK_TIME"
|
||||
}
|
||||
sync_set {
|
||||
tag_index: "START_POS"
|
||||
}
|
||||
sync_set {
|
||||
tag_index: "CANCEL_OBJECT_ID"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.BoxTrackerCalculatorOptions]: {
|
||||
tracker_options: {
|
||||
track_step_options {
|
||||
track_object_and_camera: true
|
||||
tracking_degrees: TRACKING_DEGREE_OBJECT_ROTATION_SCALE
|
||||
inlier_spring_force: 0.0
|
||||
static_motion_temporal_ratio: 3e-2
|
||||
}
|
||||
}
|
||||
visualize_tracking_data: false
|
||||
streaming_track_data_cache_size: 100
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Consolidates tracking and detection results.
|
||||
node {
|
||||
calculator: "FrameAnnotationTrackerCalculator"
|
||||
input_stream: "FRAME_ANNOTATION:objects"
|
||||
input_stream: "TRACKED_BOXES:boxes"
|
||||
output_stream: "TRACKED_FRAME_ANNOTATION:tracked_objects"
|
||||
output_stream: "CANCEL_OBJECT_ID:cancel_object_id"
|
||||
node_options: {
|
||||
[type.googleapis.com/mediapipe.FrameAnnotationTrackerCalculatorOptions] {
|
||||
img_width: 240
|
||||
img_height: 320
|
||||
}
|
||||
}
|
||||
|
||||
input_stream_handler {
|
||||
input_stream_handler: "SyncSetInputStreamHandler"
|
||||
options {
|
||||
[mediapipe.SyncSetInputStreamHandlerOptions.ext] {
|
||||
sync_set {
|
||||
tag_index: "FRAME_ANNOTATION"
|
||||
}
|
||||
sync_set {
|
||||
tag_index: "TRACKED_BOXES"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Lift the tracked 2D keypoints to 3D using EPnP algorithm.
|
||||
node {
|
||||
calculator: "Lift2DFrameAnnotationTo3DCalculator"
|
||||
input_stream: "FRAME_ANNOTATION:tracked_objects"
|
||||
output_stream: "LIFTED_FRAME_ANNOTATION:lifted_tracked_objects"
|
||||
}
|
||||
```
|
||||
@@ -61,6 +61,8 @@ videos.
|
||||
```bash
|
||||
# cd to the root directory of the MediaPipe repo
|
||||
cd -
|
||||
|
||||
pip3 install tf_slim
|
||||
python -m mediapipe.examples.desktop.youtube8m.generate_vggish_frozen_graph
|
||||
```
|
||||
|
||||
@@ -78,7 +80,7 @@ videos.
|
||||
5. Run the MediaPipe binary to extract the features.
|
||||
|
||||
```bash
|
||||
bazel build -c opt \
|
||||
bazel build -c opt --linkopt=-s \
|
||||
--define MEDIAPIPE_DISABLE_GPU=1 --define no_aws_support=true \
|
||||
mediapipe/examples/desktop/youtube8m:extract_yt8m_features
|
||||
|
||||
@@ -126,13 +128,13 @@ the inference for both local videos and the dataset
|
||||
2. Build the inference binary.
|
||||
|
||||
```bash
|
||||
bazel build -c opt --define='MEDIAPIPE_DISABLE_GPU=1' \
|
||||
bazel build -c opt --define='MEDIAPIPE_DISABLE_GPU=1' --linkopt=-s \
|
||||
mediapipe/examples/desktop/youtube8m:model_inference
|
||||
```
|
||||
|
||||
3. Run the python web server.
|
||||
|
||||
Note: pip install absl-py
|
||||
Note: pip3 install absl-py
|
||||
|
||||
```bash
|
||||
python mediapipe/examples/desktop/youtube8m/viewer/server.py --root `pwd`
|
||||
@@ -162,7 +164,7 @@ the inference for both local videos and the dataset
|
||||
3. Build and run the inference binary.
|
||||
|
||||
```bash
|
||||
bazel build -c opt --define='MEDIAPIPE_DISABLE_GPU=1' \
|
||||
bazel build -c opt --define='MEDIAPIPE_DISABLE_GPU=1' --linkopt=-s \
|
||||
mediapipe/examples/desktop/youtube8m:model_inference
|
||||
|
||||
# segment_size is the number of seconds window of frames.
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
|
||||
package="com.google.mediapipe.apps.objectdetection3d">
|
||||
|
||||
<uses-sdk
|
||||
android:minSdkVersion="21"
|
||||
android:targetSdkVersion="27" />
|
||||
|
||||
<!-- For using the camera -->
|
||||
<uses-permission android:name="android.permission.CAMERA" />
|
||||
<uses-feature android:name="android.hardware.camera" />
|
||||
<uses-feature android:name="android.hardware.camera.autofocus" />
|
||||
<!-- For MediaPipe -->
|
||||
<uses-feature android:glEsVersion="0x00020000" android:required="true" />
|
||||
|
||||
|
||||
<application
|
||||
android:allowBackup="true"
|
||||
android:label="@string/app_name"
|
||||
android:supportsRtl="true"
|
||||
android:theme="@style/AppTheme">
|
||||
<activity
|
||||
android:name=".MainActivity"
|
||||
android:exported="true"
|
||||
android:screenOrientation="portrait">
|
||||
<intent-filter>
|
||||
<action android:name="android.intent.action.MAIN" />
|
||||
<category android:name="android.intent.category.LAUNCHER" />
|
||||
</intent-filter>
|
||||
</activity>
|
||||
</application>
|
||||
|
||||
</manifest>
|
||||
@@ -0,0 +1,115 @@
|
||||
# Copyright 2019 The MediaPipe Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
licenses(["notice"]) # Apache 2.0
|
||||
|
||||
package(default_visibility = ["//visibility:private"])
|
||||
|
||||
cc_binary(
|
||||
name = "libmediapipe_jni.so",
|
||||
linkshared = 1,
|
||||
linkstatic = 1,
|
||||
deps = [
|
||||
"//mediapipe/graphs/object_detection_3d:mobile_calculators",
|
||||
"//mediapipe/java/com/google/mediapipe/framework/jni:mediapipe_framework_jni",
|
||||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "mediapipe_jni_lib",
|
||||
srcs = [":libmediapipe_jni.so"],
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
# To use the "chair" model instead of the default "shoes" model,
|
||||
# add "--define chair=true" to the bazel build command.
|
||||
config_setting(
|
||||
name = "use_chair_model",
|
||||
define_values = {
|
||||
"chair": "true",
|
||||
},
|
||||
)
|
||||
|
||||
# Maps the binary graph to an alias (e.g., the app name) for convenience so that the alias can be
|
||||
# easily incorporated into the app via, for example,
|
||||
# MainActivity.BINARY_GRAPH_NAME = "appname.binarypb".
|
||||
genrule(
|
||||
name = "binary_graph",
|
||||
srcs = select({
|
||||
"//conditions:default": ["//mediapipe/graphs/object_detection_3d:mobile_gpu_binary_graph_shoe"],
|
||||
":use_chair_model": ["//mediapipe/graphs/object_detection_3d:mobile_gpu_binary_graph_chair"],
|
||||
}),
|
||||
outs = ["objectdetection3d.binarypb"],
|
||||
cmd = "cp $< $@",
|
||||
)
|
||||
|
||||
genrule(
|
||||
name = "model",
|
||||
srcs = select({
|
||||
"//conditions:default": ["//mediapipe/models:object_detection_3d_sneakers.tflite"],
|
||||
":use_chair_model": ["//mediapipe/models:object_detection_3d_chair.tflite"],
|
||||
}),
|
||||
outs = ["object_detection_3d.tflite"],
|
||||
cmd = "cp $< $@",
|
||||
)
|
||||
|
||||
android_library(
|
||||
name = "mediapipe_lib",
|
||||
srcs = glob(["*.java"]),
|
||||
assets = [
|
||||
":binary_graph",
|
||||
":model",
|
||||
"//mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/assets:box.obj.uuu",
|
||||
"//mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/assets:classic_colors.png",
|
||||
] + select({
|
||||
"//conditions:default": [
|
||||
"//mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/assets/sneaker:model.obj.uuu",
|
||||
"//mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/assets/sneaker:texture.bmp",
|
||||
],
|
||||
":use_chair_model": [
|
||||
"//mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/assets/chair:model.obj.uuu",
|
||||
"//mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/assets/chair:texture.bmp",
|
||||
],
|
||||
}),
|
||||
assets_dir = "",
|
||||
manifest = "AndroidManifest.xml",
|
||||
resource_files = glob(["res/**"]),
|
||||
deps = [
|
||||
":mediapipe_jni_lib",
|
||||
"//mediapipe/framework/formats:landmark_java_proto_lite",
|
||||
"//mediapipe/java/com/google/mediapipe/components:android_camerax_helper",
|
||||
"//mediapipe/java/com/google/mediapipe/components:android_components",
|
||||
"//mediapipe/java/com/google/mediapipe/framework:android_framework",
|
||||
"//mediapipe/java/com/google/mediapipe/glutil",
|
||||
"//third_party:androidx_appcompat",
|
||||
"//third_party:androidx_constraint_layout",
|
||||
"//third_party:androidx_legacy_support_v4",
|
||||
"//third_party:androidx_recyclerview",
|
||||
"//third_party:opencv",
|
||||
"@androidx_concurrent_futures//jar",
|
||||
"@androidx_lifecycle//jar",
|
||||
"@com_google_code_findbugs//jar",
|
||||
"@com_google_guava_android//jar",
|
||||
],
|
||||
)
|
||||
|
||||
android_binary(
|
||||
name = "objectdetection3d",
|
||||
manifest = "AndroidManifest.xml",
|
||||
manifest_values = {"applicationId": "com.google.mediapipe.apps.objectdetection3d"},
|
||||
multidex = "native",
|
||||
deps = [
|
||||
":mediapipe_lib",
|
||||
],
|
||||
)
|
||||
@@ -0,0 +1,280 @@
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package com.google.mediapipe.apps.objectdetection3d;
|
||||
|
||||
import android.graphics.Bitmap;
|
||||
import android.graphics.BitmapFactory;
|
||||
import android.graphics.SurfaceTexture;
|
||||
import android.os.Bundle;
|
||||
import androidx.appcompat.app.AppCompatActivity;
|
||||
import android.util.Log;
|
||||
import android.util.Size;
|
||||
import android.view.SurfaceHolder;
|
||||
import android.view.SurfaceView;
|
||||
import android.view.View;
|
||||
import android.view.ViewGroup;
|
||||
import com.google.mediapipe.components.CameraHelper;
|
||||
import com.google.mediapipe.components.CameraXPreviewHelper;
|
||||
import com.google.mediapipe.components.ExternalTextureConverter;
|
||||
import com.google.mediapipe.components.FrameProcessor;
|
||||
import com.google.mediapipe.components.PermissionHelper;
|
||||
import com.google.mediapipe.framework.AndroidAssetUtil;
|
||||
import com.google.mediapipe.framework.AndroidPacketCreator;
|
||||
import com.google.mediapipe.framework.Packet;
|
||||
import com.google.mediapipe.glutil.EglManager;
|
||||
import java.io.InputStream;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
/** Main activity of MediaPipe example apps. */
|
||||
public class MainActivity extends AppCompatActivity {
|
||||
private static final String TAG = "MainActivity";
|
||||
|
||||
private static final String BINARY_GRAPH_NAME = "objectdetection3d.binarypb";
|
||||
private static final String INPUT_VIDEO_STREAM_NAME = "input_video";
|
||||
private static final String OUTPUT_VIDEO_STREAM_NAME = "output_video";
|
||||
|
||||
private static final String OBJ_TEXTURE = "texture.bmp";
|
||||
private static final String OBJ_FILE = "model.obj.uuu";
|
||||
private static final String BOX_TEXTURE = "classic_colors.png";
|
||||
private static final String BOX_FILE = "box.obj.uuu";
|
||||
|
||||
private static final CameraHelper.CameraFacing CAMERA_FACING = CameraHelper.CameraFacing.BACK;
|
||||
|
||||
// Flips the camera-preview frames vertically before sending them into FrameProcessor to be
|
||||
// processed in a MediaPipe graph, and flips the processed frames back when they are displayed.
|
||||
// This is needed because OpenGL represents images assuming the image origin is at the bottom-left
|
||||
// corner, whereas MediaPipe in general assumes the image origin is at top-left.
|
||||
private static final boolean FLIP_FRAMES_VERTICALLY = true;
|
||||
|
||||
// Target resolution should be 4:3 for this application, as expected by the model and tracker.
|
||||
private static final Size TARGET_RESOLUTION = new Size(1280, 960);
|
||||
|
||||
static {
|
||||
// Load all native libraries needed by the app.
|
||||
System.loadLibrary("mediapipe_jni");
|
||||
System.loadLibrary("opencv_java3");
|
||||
}
|
||||
|
||||
// {@link SurfaceTexture} where the camera-preview frames can be accessed.
|
||||
private SurfaceTexture previewFrameTexture;
|
||||
// {@link SurfaceView} that displays the camera-preview frames processed by a MediaPipe graph.
|
||||
private SurfaceView previewDisplayView;
|
||||
|
||||
// Creates and manages an {@link EGLContext}.
|
||||
private EglManager eglManager;
|
||||
// Sends camera-preview frames into a MediaPipe graph for processing, and displays the processed
|
||||
// frames onto a {@link Surface}.
|
||||
private FrameProcessor processor;
|
||||
// Converts the GL_TEXTURE_EXTERNAL_OES texture from Android camera into a regular texture to be
|
||||
// consumed by {@link FrameProcessor} and the underlying MediaPipe graph.
|
||||
private ExternalTextureConverter converter;
|
||||
|
||||
// Handles camera access via the {@link CameraX} Jetpack support library.
|
||||
private CameraXPreviewHelper cameraHelper;
|
||||
|
||||
// Assets.
|
||||
private Bitmap objTexture = null;
|
||||
private Bitmap boxTexture = null;
|
||||
|
||||
Size cameraImageSize;
|
||||
|
||||
@Override
|
||||
protected void onCreate(Bundle savedInstanceState) {
|
||||
super.onCreate(savedInstanceState);
|
||||
setContentView(R.layout.activity_main);
|
||||
|
||||
previewDisplayView = new SurfaceView(this);
|
||||
setupPreviewDisplayView();
|
||||
|
||||
// Initialize asset manager so that MediaPipe native libraries can access the app assets, e.g.,
|
||||
// binary graphs.
|
||||
AndroidAssetUtil.initializeNativeAssetManager(this);
|
||||
|
||||
eglManager = new EglManager(null);
|
||||
processor =
|
||||
new FrameProcessor(
|
||||
this,
|
||||
eglManager.getNativeContext(),
|
||||
BINARY_GRAPH_NAME,
|
||||
INPUT_VIDEO_STREAM_NAME,
|
||||
OUTPUT_VIDEO_STREAM_NAME);
|
||||
processor.getVideoSurfaceOutput().setFlipY(FLIP_FRAMES_VERTICALLY);
|
||||
|
||||
prepareDemoAssets();
|
||||
AndroidPacketCreator packetCreator = processor.getPacketCreator();
|
||||
Map<String, Packet> inputSidePackets = new HashMap<>();
|
||||
inputSidePackets.put("obj_asset_name", packetCreator.createString(OBJ_FILE));
|
||||
inputSidePackets.put("box_asset_name", packetCreator.createString(BOX_FILE));
|
||||
inputSidePackets.put("obj_texture", packetCreator.createRgbaImageFrame(objTexture));
|
||||
inputSidePackets.put("box_texture", packetCreator.createRgbaImageFrame(boxTexture));
|
||||
processor.setInputSidePackets(inputSidePackets);
|
||||
|
||||
PermissionHelper.checkAndRequestCameraPermissions(this);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void onResume() {
|
||||
super.onResume();
|
||||
converter = new ExternalTextureConverter(eglManager.getContext());
|
||||
converter.setFlipY(FLIP_FRAMES_VERTICALLY);
|
||||
converter.setConsumer(processor);
|
||||
if (PermissionHelper.cameraPermissionsGranted(this)) {
|
||||
startCamera();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void onPause() {
|
||||
super.onPause();
|
||||
converter.close();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onRequestPermissionsResult(
|
||||
int requestCode, String[] permissions, int[] grantResults) {
|
||||
super.onRequestPermissionsResult(requestCode, permissions, grantResults);
|
||||
PermissionHelper.onRequestPermissionsResult(requestCode, permissions, grantResults);
|
||||
}
|
||||
|
||||
private void setupPreviewDisplayView() {
|
||||
previewDisplayView.setVisibility(View.GONE);
|
||||
ViewGroup viewGroup = findViewById(R.id.preview_display_layout);
|
||||
viewGroup.addView(previewDisplayView);
|
||||
|
||||
previewDisplayView
|
||||
.getHolder()
|
||||
.addCallback(
|
||||
new SurfaceHolder.Callback() {
|
||||
@Override
|
||||
public void surfaceCreated(SurfaceHolder holder) {
|
||||
processor.getVideoSurfaceOutput().setSurface(holder.getSurface());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void surfaceChanged(SurfaceHolder holder, int format, int width, int height) {
|
||||
// (Re-)Compute the ideal size of the camera-preview display (the area that the
|
||||
// camera-preview frames get rendered onto, potentially with scaling and rotation)
|
||||
// based on the size of the SurfaceView that contains the display.
|
||||
Size viewSize = new Size(height, height * 3 / 4); // Prefer 3:4 aspect ratio.
|
||||
Size displaySize = cameraHelper.computeDisplaySizeFromViewSize(viewSize);
|
||||
boolean isCameraRotated = cameraHelper.isCameraRotated();
|
||||
cameraImageSize = cameraHelper.getFrameSize();
|
||||
|
||||
// Connect the converter to the camera-preview frames as its input (via
|
||||
// previewFrameTexture), and configure the output width and height as the computed
|
||||
// display size.
|
||||
converter.setSurfaceTextureAndAttachToGLContext(
|
||||
previewFrameTexture,
|
||||
isCameraRotated ? displaySize.getHeight() : displaySize.getWidth(),
|
||||
isCameraRotated ? displaySize.getWidth() : displaySize.getHeight());
|
||||
processor.setOnWillAddFrameListener(
|
||||
(timestamp) -> {
|
||||
try {
|
||||
int cameraTextureWidth =
|
||||
isCameraRotated
|
||||
? cameraImageSize.getHeight()
|
||||
: cameraImageSize.getWidth();
|
||||
int cameraTextureHeight =
|
||||
isCameraRotated
|
||||
? cameraImageSize.getWidth()
|
||||
: cameraImageSize.getHeight();
|
||||
|
||||
// Find limiting side and scale to 3:4 aspect ratio
|
||||
float aspectRatio =
|
||||
(float) cameraTextureWidth / (float) cameraTextureHeight;
|
||||
if (aspectRatio > 3.0 / 4.0) {
|
||||
// width too big
|
||||
cameraTextureWidth = (int) ((float) cameraTextureHeight * 3.0 / 4.0);
|
||||
} else {
|
||||
// height too big
|
||||
cameraTextureHeight = (int) ((float) cameraTextureWidth * 4.0 / 3.0);
|
||||
}
|
||||
Packet widthPacket =
|
||||
processor.getPacketCreator().createInt32(cameraTextureWidth);
|
||||
Packet heightPacket =
|
||||
processor.getPacketCreator().createInt32(cameraTextureHeight);
|
||||
|
||||
try {
|
||||
processor
|
||||
.getGraph()
|
||||
.addPacketToInputStream("input_width", widthPacket, timestamp);
|
||||
processor
|
||||
.getGraph()
|
||||
.addPacketToInputStream("input_height", heightPacket, timestamp);
|
||||
} catch (Exception e) {
|
||||
Log.e(
|
||||
TAG,
|
||||
"MediaPipeException encountered adding packets to width and height"
|
||||
+ " input streams.");
|
||||
}
|
||||
widthPacket.release();
|
||||
heightPacket.release();
|
||||
} catch (IllegalStateException ise) {
|
||||
Log.e(
|
||||
TAG,
|
||||
"Exception while adding packets to width and height input streams.");
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@Override
|
||||
public void surfaceDestroyed(SurfaceHolder holder) {
|
||||
processor.getVideoSurfaceOutput().setSurface(null);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private void startCamera() {
|
||||
cameraHelper = new CameraXPreviewHelper();
|
||||
cameraHelper.setOnCameraStartedListener(
|
||||
surfaceTexture -> {
|
||||
previewFrameTexture = surfaceTexture;
|
||||
// Make the display view visible to start showing the preview. This triggers the
|
||||
// SurfaceHolder.Callback added to (the holder of) previewDisplayView.
|
||||
previewDisplayView.setVisibility(View.VISIBLE);
|
||||
});
|
||||
cameraHelper.startCamera(
|
||||
this, CAMERA_FACING, /*surfaceTexture=*/ null, /*targetSize=*/ TARGET_RESOLUTION);
|
||||
cameraImageSize = cameraHelper.getFrameSize();
|
||||
}
|
||||
|
||||
private void prepareDemoAssets() {
|
||||
AndroidAssetUtil.initializeNativeAssetManager(this);
|
||||
// We render from raw data with openGL, so disable decoding preprocessing
|
||||
BitmapFactory.Options decodeOptions = new BitmapFactory.Options();
|
||||
decodeOptions.inScaled = false;
|
||||
decodeOptions.inDither = false;
|
||||
decodeOptions.inPremultiplied = false;
|
||||
|
||||
try {
|
||||
InputStream inputStream = getAssets().open(OBJ_TEXTURE);
|
||||
objTexture = BitmapFactory.decodeStream(inputStream, null /*outPadding*/, decodeOptions);
|
||||
inputStream.close();
|
||||
} catch (Exception e) {
|
||||
Log.e(TAG, "Error parsing object texture; error: " + e);
|
||||
throw new IllegalStateException(e);
|
||||
}
|
||||
|
||||
try {
|
||||
InputStream inputStream = getAssets().open(BOX_TEXTURE);
|
||||
boxTexture = BitmapFactory.decodeStream(inputStream, null /*outPadding*/, decodeOptions);
|
||||
inputStream.close();
|
||||
} catch (Exception e) {
|
||||
Log.e(TAG, "Error parsing box texture; error: " + e);
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
# Copyright 2019 The MediaPipe Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
licenses(["notice"]) # Apache 2.0
|
||||
|
||||
package(default_visibility = ["//visibility:public"])
|
||||
|
||||
exports_files(
|
||||
srcs = glob(["**"]),
|
||||
)
|
||||
@@ -0,0 +1,21 @@
|
||||
# Copyright 2019 The MediaPipe Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
licenses(["notice"]) # Apache 2.0
|
||||
|
||||
package(default_visibility = ["//visibility:public"])
|
||||
|
||||
exports_files(
|
||||
srcs = glob(["**"]),
|
||||
)
|
||||
|
After Width: | Height: | Size: 6.9 MiB |
|
After Width: | Height: | Size: 17 KiB |
|
After Width: | Height: | Size: 410 KiB |
@@ -0,0 +1,21 @@
|
||||
# Copyright 2019 The MediaPipe Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
licenses(["notice"]) # Apache 2.0
|
||||
|
||||
package(default_visibility = ["//visibility:public"])
|
||||
|
||||
exports_files(
|
||||
srcs = glob(["**"]),
|
||||
)
|
||||
|
After Width: | Height: | Size: 48 MiB |
@@ -0,0 +1,20 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<androidx.constraintlayout.widget.ConstraintLayout xmlns:android="http://schemas.android.com/apk/res/android"
|
||||
xmlns:app="http://schemas.android.com/apk/res-auto"
|
||||
xmlns:tools="http://schemas.android.com/tools"
|
||||
android:layout_width="match_parent"
|
||||
android:layout_height="match_parent">
|
||||
|
||||
<FrameLayout
|
||||
android:id="@+id/preview_display_layout"
|
||||
android:layout_width="fill_parent"
|
||||
android:layout_height="fill_parent"
|
||||
android:layout_weight="1">
|
||||
<TextView
|
||||
android:id="@+id/no_camera_access_view"
|
||||
android:layout_height="fill_parent"
|
||||
android:layout_width="fill_parent"
|
||||
android:gravity="center"
|
||||
android:text="@string/no_camera_access" />
|
||||
</FrameLayout>
|
||||
</androidx.constraintlayout.widget.ConstraintLayout>
|
||||
@@ -0,0 +1,6 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<resources>
|
||||
<color name="colorPrimary">#008577</color>
|
||||
<color name="colorPrimaryDark">#00574B</color>
|
||||
<color name="colorAccent">#D81B60</color>
|
||||
</resources>
|
||||
@@ -0,0 +1,4 @@
|
||||
<resources>
|
||||
<string name="app_name" translatable="false">Object Detection 3D</string>
|
||||
<string name="no_camera_access" translatable="false">Please grant camera permissions.</string>
|
||||
</resources>
|
||||
@@ -0,0 +1,11 @@
|
||||
<resources>
|
||||
|
||||
<!-- Base application theme. -->
|
||||
<style name="AppTheme" parent="Theme.AppCompat.Light.DarkActionBar">
|
||||
<!-- Customize your theme here. -->
|
||||
<item name="colorPrimary">@color/colorPrimary</item>
|
||||
<item name="colorPrimaryDark">@color/colorPrimaryDark</item>
|
||||
<item name="colorAccent">@color/colorAccent</item>
|
||||
</style>
|
||||
|
||||
</resources>
|
||||
@@ -63,7 +63,7 @@ COPY . /mediapipe/
|
||||
|
||||
# Install bazel
|
||||
|
||||
ARG BAZEL_VERSION=0.29.1
|
||||
ARG BAZEL_VERSION=1.1.0
|
||||
RUN mkdir /bazel && \
|
||||
wget --no-check-certificate -O /bazel/installer.sh "https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh" && \
|
||||
wget --no-check-certificate -O /bazel/LICENSE.txt "https://raw.githubusercontent.com/bazelbuild/bazel/master/LICENSE" && \
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
# Coral Dev Board Setup (experimental)
|
||||
|
||||
**Dislaimer**: Running MediaPipe on Coral is experimental, and this process may
|
||||
**Disclaimer**: Running MediaPipe on Coral is experimental, and this process may
|
||||
not be exact and is subject to change. These instructions have only been tested
|
||||
on the [Coral Dev Board](https://coral.ai/products/dev-board/) with Mendel 4.0,
|
||||
and may vary for different devices and workstations.
|
||||
on the [Coral Dev Board](https://coral.ai/products/dev-board/)
|
||||
running [Mendel Enterprise Day 13](https://coral.ai/software/) OS and
|
||||
using [Diploria2](https://github.com/google-coral/edgetpu/tree/diploria2)
|
||||
edgetpu libs, and may vary for different devices and workstations.
|
||||
|
||||
This file describes how to prepare a Coral Dev Board and setup a Linux
|
||||
Docker container for building MediaPipe applications that run on Edge TPU.
|
||||
@@ -16,10 +18,12 @@ Docker container for building MediaPipe applications that run on Edge TPU.
|
||||
|
||||
* Setup the coral device via [here](https://coral.withgoogle.com/docs/dev-board/get-started/), and ensure the _mdt_ command works
|
||||
|
||||
Note: alias mdt="python3 -m mdt.main" may be needed on some systems
|
||||
|
||||
* (on coral device) prepare MediaPipe
|
||||
|
||||
cd ~
|
||||
sudo apt-get install -y git
|
||||
sudo apt-get update && sudo apt-get install -y git
|
||||
git clone https://github.com/google/mediapipe.git
|
||||
mkdir mediapipe/bazel-bin
|
||||
|
||||
|
||||
@@ -10,19 +10,25 @@ http_archive(
|
||||
sha256 = "2ef429f5d7ce7111263289644d233707dba35e39696377ebab8b0bc701f7818e",
|
||||
)
|
||||
load("@bazel_skylib//lib:versions.bzl", "versions")
|
||||
versions.check(minimum_bazel_version = "0.24.1")
|
||||
versions.check(minimum_bazel_version = "1.0.0",
|
||||
maximum_bazel_version = "1.2.1")
|
||||
|
||||
# ABSL cpp library.
|
||||
|
||||
# ABSL cpp library lts_2020_02_25
|
||||
http_archive(
|
||||
name = "com_google_absl",
|
||||
# Head commit on 2019-04-12.
|
||||
# TODO: Switch to the latest absl version when the problem gets
|
||||
# fixed.
|
||||
urls = [
|
||||
"https://github.com/abseil/abseil-cpp/archive/a02f62f456f2c4a7ecf2be3104fe0c6e16fbad9a.tar.gz",
|
||||
"https://github.com/abseil/abseil-cpp/archive/20200225.tar.gz",
|
||||
],
|
||||
sha256 = "d437920d1434c766d22e85773b899c77c672b8b4865d5dc2cd61a29fdff3cf03",
|
||||
strip_prefix = "abseil-cpp-a02f62f456f2c4a7ecf2be3104fe0c6e16fbad9a",
|
||||
# Remove after https://github.com/abseil/abseil-cpp/issues/326 is solved.
|
||||
patches = [
|
||||
"@//third_party:com_google_absl_f863b622fe13612433fdf43f76547d5edda0c93001.diff"
|
||||
],
|
||||
patch_args = [
|
||||
"-p1",
|
||||
],
|
||||
strip_prefix = "abseil-cpp-20200225",
|
||||
sha256 = "728a813291bdec2aa46eab8356ace9f75ac2ed9dfe2df5ab603c4e6c09f1c353"
|
||||
)
|
||||
|
||||
http_archive(
|
||||
@@ -72,6 +78,14 @@ http_archive(
|
||||
],
|
||||
)
|
||||
|
||||
# easyexif
|
||||
http_archive(
|
||||
name = "easyexif",
|
||||
url = "https://github.com/mayanklahiri/easyexif/archive/master.zip",
|
||||
strip_prefix = "easyexif-master",
|
||||
build_file = "@//third_party:easyexif.BUILD",
|
||||
)
|
||||
|
||||
# libyuv
|
||||
http_archive(
|
||||
name = "libyuv",
|
||||
@@ -103,15 +117,23 @@ http_archive(
|
||||
],
|
||||
)
|
||||
|
||||
# 2019-11-12
|
||||
_TENSORFLOW_GIT_COMMIT = "a5f9bcd64453ff3d1f64cb4da4786db3d2da7f82"
|
||||
_TENSORFLOW_SHA256= "f2b6f2ab2ffe63e86eccd3ce4bea6b7197383d726638dfeeebcdc1e7de73f075"
|
||||
# 2020-02-12
|
||||
# The last commit before TensorFlow switched to Bazel 2.0
|
||||
_TENSORFLOW_GIT_COMMIT = "77e9ffb9b2bfb1a4f7056e62d84039626923e328"
|
||||
_TENSORFLOW_SHA256= "176ccd82f7dd17c5e117b50d353603b129c7a6ccbfebd522ca47cc2a40f33f13"
|
||||
http_archive(
|
||||
name = "org_tensorflow",
|
||||
urls = [
|
||||
"https://mirror.bazel.build/github.com/tensorflow/tensorflow/archive/%s.tar.gz" % _TENSORFLOW_GIT_COMMIT,
|
||||
"https://github.com/tensorflow/tensorflow/archive/%s.tar.gz" % _TENSORFLOW_GIT_COMMIT,
|
||||
],
|
||||
# A compatibility patch
|
||||
patches = [
|
||||
"@//third_party:org_tensorflow_528e22eae8bf3206189a066032c66e9e5c9b4a61.diff"
|
||||
],
|
||||
patch_args = [
|
||||
"-p1",
|
||||
],
|
||||
strip_prefix = "tensorflow-%s" % _TENSORFLOW_GIT_COMMIT,
|
||||
sha256 = _TENSORFLOW_SHA256,
|
||||
)
|
||||
@@ -119,8 +141,22 @@ http_archive(
|
||||
load("@org_tensorflow//tensorflow:workspace.bzl", "tf_workspace")
|
||||
tf_workspace(tf_repo_name = "org_tensorflow")
|
||||
|
||||
http_archive(
|
||||
name = "ceres_solver",
|
||||
url = "https://github.com/ceres-solver/ceres-solver/archive/1.14.0.zip",
|
||||
patches = [
|
||||
"@//third_party:ceres_solver_9bf9588988236279e1262f75d7f4d85711dfa172.diff"
|
||||
],
|
||||
patch_args = [
|
||||
"-p1",
|
||||
],
|
||||
strip_prefix = "ceres-solver-1.14.0",
|
||||
sha256 = "5ba6d0db4e784621fda44a50c58bb23b0892684692f0c623e2063f9c19f192f1"
|
||||
)
|
||||
|
||||
# Please run
|
||||
# $ sudo apt-get install libopencv-core-dev libopencv-highgui-dev \
|
||||
# libopencv-calib3d-dev libopencv-features2d-dev \
|
||||
# libopencv-imgproc-dev libopencv-video-dev
|
||||
new_local_repository(
|
||||
name = "linux_opencv",
|
||||
@@ -149,11 +185,10 @@ new_local_repository(
|
||||
|
||||
http_archive(
|
||||
name = "android_opencv",
|
||||
sha256 = "056b849842e4fa8751d09edbb64530cfa7a63c84ccd232d0ace330e27ba55d0b",
|
||||
build_file = "@//third_party:opencv_android.BUILD",
|
||||
strip_prefix = "OpenCV-android-sdk",
|
||||
type = "zip",
|
||||
url = "https://github.com/opencv/opencv/releases/download/4.1.0/opencv-4.1.0-android-sdk.zip",
|
||||
url = "https://github.com/opencv/opencv/releases/download/3.4.3/opencv-3.4.3-android-sdk.zip",
|
||||
)
|
||||
|
||||
# After OpenCV 3.2.0, the pre-compiled opencv2.framework has google protobuf symbols, which will
|
||||
@@ -184,13 +219,18 @@ maven_install(
|
||||
artifacts = [
|
||||
"androidx.annotation:annotation:aar:1.1.0",
|
||||
"androidx.appcompat:appcompat:aar:1.1.0-rc01",
|
||||
"androidx.camera:camera-core:aar:1.0.0-alpha06",
|
||||
"androidx.camera:camera-camera2:aar:1.0.0-alpha06",
|
||||
"androidx.constraintlayout:constraintlayout:aar:1.1.3",
|
||||
"androidx.core:core:aar:1.1.0-rc03",
|
||||
"androidx.legacy:legacy-support-v4:aar:1.0.0",
|
||||
"androidx.recyclerview:recyclerview:aar:1.1.0-beta02",
|
||||
"com.google.android.material:material:aar:1.0.0-rc01",
|
||||
],
|
||||
repositories = ["https://dl.google.com/dl/android/maven2"],
|
||||
repositories = [
|
||||
"https://dl.google.com/dl/android/maven2",
|
||||
"https://repo1.maven.org/maven2",
|
||||
],
|
||||
)
|
||||
|
||||
maven_server(
|
||||
@@ -206,10 +246,10 @@ maven_jar(
|
||||
)
|
||||
|
||||
maven_jar(
|
||||
name = "androidx_concurrent_futures",
|
||||
artifact = "androidx.concurrent:concurrent-futures:1.0.0-alpha03",
|
||||
sha1 = "b528df95c7e2fefa2210c0c742bf3e491c1818ae",
|
||||
server = "google_server",
|
||||
name = "androidx_concurrent_futures",
|
||||
artifact = "androidx.concurrent:concurrent-futures:1.0.0-alpha03",
|
||||
sha1 = "b528df95c7e2fefa2210c0c742bf3e491c1818ae",
|
||||
server = "google_server",
|
||||
)
|
||||
|
||||
maven_jar(
|
||||
@@ -285,10 +325,13 @@ http_archive(
|
||||
build_file = "@//third_party:google_toolbox_for_mac.BUILD",
|
||||
)
|
||||
|
||||
### Coral ###
|
||||
|
||||
# Coral
|
||||
#COMMIT=$(git ls-remote https://github.com/google-coral/crosstool master | awk '{print $1}')
|
||||
#SHA256=$(curl -L "https://github.com/google-coral/crosstool/archive/${COMMIT}.tar.gz" | sha256sum | awk '{print $1}')
|
||||
# Oct 2019
|
||||
#COMMIT=9e00d5be43bf001f883b5700f5d04882fea00229
|
||||
#SHA256=cb31b1417ccdcf7dd9fca5ec63e1571672372c30427730255997a547569d2feb
|
||||
http_archive(
|
||||
name = "coral_crosstool",
|
||||
sha256 = "cb31b1417ccdcf7dd9fca5ec63e1571672372c30427730255997a547569d2feb",
|
||||
|
||||
@@ -182,8 +182,8 @@ node {
|
||||
# Draws annotations and overlays them on top of the input images.
|
||||
node {
|
||||
calculator: "AnnotationOverlayCalculator"
|
||||
input_stream: "INPUT_FRAME:throttled_input_video"
|
||||
input_stream: "IMAGE:throttled_input_video"
|
||||
input_stream: "render_data"
|
||||
output_stream: "OUTPUT_FRAME:output_video"
|
||||
output_stream: "IMAGE:output_video"
|
||||
}
|
||||
|
||||
|
||||
@@ -173,7 +173,7 @@ node {
|
||||
# Draws annotations and overlays them on top of the input images.
|
||||
node {
|
||||
calculator: "AnnotationOverlayCalculator"
|
||||
input_stream: "INPUT_FRAME:throttled_input_video"
|
||||
input_stream: "IMAGE:throttled_input_video"
|
||||
input_stream: "render_data"
|
||||
output_stream: "OUTPUT_FRAME:output_video"
|
||||
output_stream: "IMAGE:output_video"
|
||||
}
|
||||
|
||||
@@ -8,7 +8,7 @@ echo ' sh mediapipe/examples/coral/setup.sh '
|
||||
|
||||
sleep 3
|
||||
|
||||
mkdir opencv32_arm64_libs
|
||||
mkdir -p opencv32_arm64_libs
|
||||
|
||||
cp mediapipe/examples/coral/update_sources.sh update_sources.sh
|
||||
chmod +x update_sources.sh
|
||||
|
||||
@@ -53,7 +53,7 @@ cc_library(
|
||||
|
||||
# Linux only.
|
||||
# Must have a GPU with EGL support:
|
||||
# ex: sudo aptitude install mesa-common-dev libegl1-mesa-dev libgles2-mesa-dev
|
||||
# ex: sudo apt-get install mesa-common-dev libegl1-mesa-dev libgles2-mesa-dev
|
||||
# (or similar nvidia/amd equivalent)
|
||||
cc_library(
|
||||
name = "demo_run_graph_main_gpu",
|
||||
|
||||
@@ -11,6 +11,8 @@
|
||||
|
||||
2. Build and run the run_autoflip binary to process a local video.
|
||||
|
||||
Note: AutoFlip currently only works with OpenCV 3 . Please verify your OpenCV version beforehand.
|
||||
|
||||
```bash
|
||||
bazel build -c opt --define MEDIAPIPE_DISABLE_GPU=1 \
|
||||
mediapipe/examples/desktop/autoflip:run_autoflip
|
||||
|
||||
@@ -63,12 +63,15 @@ import random
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import urllib
|
||||
import zipfile
|
||||
|
||||
from absl import app
|
||||
from absl import flags
|
||||
from absl import logging
|
||||
import tensorflow as tf
|
||||
from six.moves import range
|
||||
from six.moves import urllib
|
||||
import tensorflow.compat.v1 as tf
|
||||
|
||||
from mediapipe.util.sequence import media_sequence as ms
|
||||
|
||||
|
||||
@@ -218,7 +221,7 @@ class Charades(object):
|
||||
return output_dict
|
||||
|
||||
if split not in SPLITS:
|
||||
raise ValueError("Split %s not in %s" % split, str(SPLITS.keys()))
|
||||
raise ValueError("Split %s not in %s" % split, str(list(SPLITS.keys())))
|
||||
all_shards = tf.io.gfile.glob(
|
||||
os.path.join(self.path_to_data, SPLITS[split][0] + "-*-of-*"))
|
||||
random.shuffle(all_shards)
|
||||
@@ -329,7 +332,7 @@ class Charades(object):
|
||||
if sys.version_info >= (3, 0):
|
||||
urlretrieve = urllib.request.urlretrieve
|
||||
else:
|
||||
urlretrieve = urllib.urlretrieve
|
||||
urlretrieve = urllib.request.urlretrieve
|
||||
logging.info("Creating data directory.")
|
||||
tf.io.gfile.makedirs(self.path_to_data)
|
||||
logging.info("Downloading license.")
|
||||
|
||||
@@ -57,12 +57,13 @@ import random
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import urllib
|
||||
|
||||
from absl import app
|
||||
from absl import flags
|
||||
from absl import logging
|
||||
import tensorflow as tf
|
||||
from six.moves import range
|
||||
from six.moves import urllib
|
||||
import tensorflow.compat.v1 as tf
|
||||
|
||||
from mediapipe.util.sequence import media_sequence as ms
|
||||
|
||||
@@ -198,7 +199,7 @@ class DemoDataset(object):
|
||||
if sys.version_info >= (3, 0):
|
||||
urlretrieve = urllib.request.urlretrieve
|
||||
else:
|
||||
urlretrieve = urllib.urlretrieve
|
||||
urlretrieve = urllib.request.urlretrieve
|
||||
for split in SPLITS:
|
||||
reader = csv.DictReader(SPLITS[split].split("\n"))
|
||||
all_metadata = []
|
||||
|
||||
@@ -73,12 +73,14 @@ import subprocess
|
||||
import sys
|
||||
import tarfile
|
||||
import tempfile
|
||||
import urllib
|
||||
|
||||
from absl import app
|
||||
from absl import flags
|
||||
from absl import logging
|
||||
import tensorflow as tf
|
||||
from six.moves import range
|
||||
from six.moves import urllib
|
||||
from six.moves import zip
|
||||
import tensorflow.compat.v1 as tf
|
||||
|
||||
from mediapipe.util.sequence import media_sequence as ms
|
||||
|
||||
@@ -96,15 +98,15 @@ FILEPATTERN = "kinetics_700_%s_25fps_rgb_flow"
|
||||
SPLITS = {
|
||||
"train": {
|
||||
"shards": 1000,
|
||||
"examples": 540247
|
||||
"examples": 538779
|
||||
},
|
||||
"validate": {
|
||||
"shards": 100,
|
||||
"examples": 34610
|
||||
"examples": 34499
|
||||
},
|
||||
"test": {
|
||||
"shards": 100,
|
||||
"examples": 69103
|
||||
"examples": 68847
|
||||
},
|
||||
"custom": {
|
||||
"csv": None, # Add a CSV for your own data here.
|
||||
@@ -198,7 +200,7 @@ class Kinetics(object):
|
||||
return output_dict
|
||||
|
||||
if split not in SPLITS:
|
||||
raise ValueError("Split %s not in %s" % split, str(SPLITS.keys()))
|
||||
raise ValueError("Split %s not in %s" % split, str(list(SPLITS.keys())))
|
||||
all_shards = tf.io.gfile.glob(
|
||||
os.path.join(self.path_to_data, FILEPATTERN % split + "-*-of-*"))
|
||||
random.shuffle(all_shards)
|
||||
@@ -302,11 +304,12 @@ class Kinetics(object):
|
||||
continue
|
||||
# rename the row with a constitent set of names.
|
||||
if len(csv_row) == 5:
|
||||
row = dict(zip(["label_name", "video", "start", "end", "split"],
|
||||
csv_row))
|
||||
row = dict(
|
||||
list(
|
||||
zip(["label_name", "video", "start", "end", "split"],
|
||||
csv_row)))
|
||||
else:
|
||||
row = dict(zip(["video", "start", "end", "split"],
|
||||
csv_row))
|
||||
row = dict(list(zip(["video", "start", "end", "split"], csv_row)))
|
||||
metadata = tf.train.SequenceExample()
|
||||
ms.set_example_id(bytes23(row["video"] + "_" + row["start"]),
|
||||
metadata)
|
||||
@@ -328,7 +331,7 @@ class Kinetics(object):
|
||||
if sys.version_info >= (3, 0):
|
||||
urlretrieve = urllib.request.urlretrieve
|
||||
else:
|
||||
urlretrieve = urllib.urlretrieve
|
||||
urlretrieve = urllib.request.urlretrieve
|
||||
logging.info("Creating data directory.")
|
||||
tf.io.gfile.makedirs(self.path_to_data)
|
||||
logging.info("Downloading annotations.")
|
||||
@@ -404,7 +407,7 @@ class Kinetics(object):
|
||||
assert NUM_CLASSES == num_keys, (
|
||||
"Found %d labels for split: %s, should be %d" % (
|
||||
num_keys, name, NUM_CLASSES))
|
||||
label_map = dict(zip(classes, range(len(classes))))
|
||||
label_map = dict(list(zip(classes, list(range(len(classes))))))
|
||||
if SPLITS[name]["examples"] > 0:
|
||||
assert SPLITS[name]["examples"] == num_examples, (
|
||||
"Found %d examples for split: %s, should be %d" % (
|
||||
|
||||
@@ -30,6 +30,8 @@
|
||||
```bash
|
||||
# cd to the root directory of the MediaPipe repo
|
||||
cd -
|
||||
|
||||
pip3 install tf_slim
|
||||
python -m mediapipe.examples.desktop.youtube8m.generate_vggish_frozen_graph
|
||||
```
|
||||
|
||||
@@ -47,7 +49,7 @@
|
||||
5. Run the MediaPipe binary to extract the features.
|
||||
|
||||
```bash
|
||||
bazel build -c opt \
|
||||
bazel build -c opt --linkopt=-s \
|
||||
--define MEDIAPIPE_DISABLE_GPU=1 --define no_aws_support=true \
|
||||
mediapipe/examples/desktop/youtube8m:extract_yt8m_features
|
||||
|
||||
@@ -87,7 +89,7 @@
|
||||
3. Build and run the inference binary.
|
||||
|
||||
```bash
|
||||
bazel build -c opt --define='MEDIAPIPE_DISABLE_GPU=1' \
|
||||
bazel build -c opt --define='MEDIAPIPE_DISABLE_GPU=1' --linkopt=-s \
|
||||
mediapipe/examples/desktop/youtube8m:model_inference
|
||||
|
||||
GLOG_logtostderr=1 bazel-bin/mediapipe/examples/desktop/youtube8m/model_inference \
|
||||
@@ -113,13 +115,13 @@
|
||||
2. Build the inference binary.
|
||||
|
||||
```bash
|
||||
bazel build -c opt --define='MEDIAPIPE_DISABLE_GPU=1' \
|
||||
bazel build -c opt --define='MEDIAPIPE_DISABLE_GPU=1' --linkopt=-s \
|
||||
mediapipe/examples/desktop/youtube8m:model_inference
|
||||
```
|
||||
|
||||
3. Run the python web server.
|
||||
|
||||
Note: pip install absl-py
|
||||
Note: pip3 install absl-py
|
||||
|
||||
```bash
|
||||
python mediapipe/examples/desktop/youtube8m/viewer/server.py --root `pwd`
|
||||
@@ -142,7 +144,7 @@
|
||||
3. Build and run the inference binary.
|
||||
|
||||
```bash
|
||||
bazel build -c opt --define='MEDIAPIPE_DISABLE_GPU=1' \
|
||||
bazel build -c opt --define='MEDIAPIPE_DISABLE_GPU=1' --linkopt=-s \
|
||||
mediapipe/examples/desktop/youtube8m:model_inference
|
||||
|
||||
# segment_size is the number of seconds window of frames.
|
||||
|
||||
@@ -21,6 +21,7 @@ import sys
|
||||
|
||||
from absl import app
|
||||
from absl import flags
|
||||
import six
|
||||
import tensorflow.compat.v1 as tf
|
||||
from mediapipe.util.sequence import media_sequence as ms
|
||||
|
||||
@@ -54,7 +55,7 @@ def main(argv):
|
||||
ms.set_clip_end_timestamp(
|
||||
flags.FLAGS.clip_end_time_sec * SECONDS_TO_MICROSECONDS, metadata)
|
||||
with open('/tmp/mediapipe/metadata.pb', 'wb') as writer:
|
||||
writer.write(metadata.SerializeToString())
|
||||
writer.write(six.ensure_binary(metadata.SerializeToString()))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
@@ -25,7 +25,7 @@ import sys
|
||||
|
||||
from absl import app
|
||||
import tensorflow.compat.v1 as tf
|
||||
from tensorflow.compat.v1.python.tools import freeze_graph
|
||||
from tensorflow.python.tools import freeze_graph
|
||||
|
||||
BASE_DIR = '/tmp/mediapipe/'
|
||||
|
||||
|
||||
@@ -1078,10 +1078,16 @@ cc_library(
|
||||
cc_library(
|
||||
name = "port",
|
||||
hdrs = ["port.h"],
|
||||
defines = select({
|
||||
"//conditions:default": [],
|
||||
}) + select({
|
||||
"//conditions:default": [],
|
||||
"//mediapipe/gpu:disable_gpu": ["MEDIAPIPE_DISABLE_GPU"],
|
||||
}),
|
||||
visibility = [
|
||||
"//mediapipe/framework:__subpackages__",
|
||||
"//mediapipe/framework/port:__pkg__",
|
||||
"//mediapipe/util:__pkg__",
|
||||
"//mediapipe/util:__subpackages__",
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
@@ -1301,7 +1301,7 @@ void PrintTimingToInfo(const std::string& label, int64 timer_value) {
|
||||
"%02lld days, %02lld:%02lld:%02lld.%03lld (total seconds: "
|
||||
"%lld.%06lld)",
|
||||
days, hours, minutes, seconds, milliseconds, total_seconds,
|
||||
timer_value % 1000000ll);
|
||||
timer_value % int64{1000000});
|
||||
}
|
||||
|
||||
bool MetricElementComparator(const std::pair<std::string, int64>& e1,
|
||||
|
||||
@@ -251,6 +251,7 @@ cc_library(
|
||||
"//mediapipe/framework/port:logging",
|
||||
"@com_google_absl//absl/base:core_headers",
|
||||
"@com_google_absl//absl/memory",
|
||||
"@com_google_absl//absl/status",
|
||||
"@com_google_absl//absl/strings",
|
||||
],
|
||||
)
|
||||
|
||||
@@ -105,6 +105,30 @@ namespace file {
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status MatchFileTypeInDirectory(
|
||||
const std::string& directory, const std::string& file_suffix,
|
||||
std::vector<std::string>* results) {
|
||||
DIR* dir = opendir(directory.c_str());
|
||||
CHECK(dir);
|
||||
// Iterates through the direcotry.
|
||||
while (true) {
|
||||
struct dirent* dir_ent = readdir(dir);
|
||||
if (dir_ent == nullptr) {
|
||||
break;
|
||||
}
|
||||
if (std::string(dir_ent->d_name) == "." ||
|
||||
std::string(dir_ent->d_name) == "..") {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (absl::EndsWith(std::string(dir_ent->d_name), file_suffix)) {
|
||||
results->push_back(JoinPath(directory, std::string(dir_ent->d_name)));
|
||||
}
|
||||
}
|
||||
closedir(dir);
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status Exists(absl::string_view file_name) {
|
||||
struct stat buffer;
|
||||
int status;
|
||||
|
||||
@@ -30,6 +30,10 @@ namespace file {
|
||||
const std::string& parent_directory, const std::string& file_name,
|
||||
std::vector<std::string>* results);
|
||||
|
||||
::mediapipe::Status MatchFileTypeInDirectory(const std::string& directory,
|
||||
const std::string& file_suffix,
|
||||
std::vector<std::string>* results);
|
||||
|
||||
::mediapipe::Status Exists(absl::string_view file_name);
|
||||
|
||||
} // namespace file
|
||||
|
||||
@@ -18,103 +18,6 @@
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
Status::Status(::mediapipe::StatusCode code, absl::string_view msg) {
|
||||
state_ = std::unique_ptr<State>(new State);
|
||||
state_->code = code;
|
||||
state_->msg = std::string(msg);
|
||||
}
|
||||
|
||||
void Status::Update(const Status& new_status) {
|
||||
if (ok()) {
|
||||
*this = new_status;
|
||||
}
|
||||
}
|
||||
|
||||
void Status::SlowCopyFrom(const State* src) {
|
||||
if (src == nullptr) {
|
||||
state_ = nullptr;
|
||||
} else {
|
||||
state_ = std::unique_ptr<State>(new State(*src));
|
||||
}
|
||||
}
|
||||
|
||||
const std::string& Status::empty_string() {
|
||||
static std::string* empty = new std::string;
|
||||
return *empty;
|
||||
}
|
||||
|
||||
std::string Status::ToString() const {
|
||||
if (state_ == nullptr) {
|
||||
return "OK";
|
||||
} else {
|
||||
char tmp[30];
|
||||
const char* type;
|
||||
switch (code()) {
|
||||
case ::mediapipe::StatusCode::kCancelled:
|
||||
type = "Cancelled";
|
||||
break;
|
||||
case ::mediapipe::StatusCode::kUnknown:
|
||||
type = "Unknown";
|
||||
break;
|
||||
case ::mediapipe::StatusCode::kInvalidArgument:
|
||||
type = "Invalid argument";
|
||||
break;
|
||||
case ::mediapipe::StatusCode::kDeadlineExceeded:
|
||||
type = "Deadline exceeded";
|
||||
break;
|
||||
case ::mediapipe::StatusCode::kNotFound:
|
||||
type = "Not found";
|
||||
break;
|
||||
case ::mediapipe::StatusCode::kAlreadyExists:
|
||||
type = "Already exists";
|
||||
break;
|
||||
case ::mediapipe::StatusCode::kPermissionDenied:
|
||||
type = "Permission denied";
|
||||
break;
|
||||
case ::mediapipe::StatusCode::kUnauthenticated:
|
||||
type = "Unauthenticated";
|
||||
break;
|
||||
case ::mediapipe::StatusCode::kResourceExhausted:
|
||||
type = "Resource exhausted";
|
||||
break;
|
||||
case ::mediapipe::StatusCode::kFailedPrecondition:
|
||||
type = "Failed precondition";
|
||||
break;
|
||||
case ::mediapipe::StatusCode::kAborted:
|
||||
type = "Aborted";
|
||||
break;
|
||||
case ::mediapipe::StatusCode::kOutOfRange:
|
||||
type = "Out of range";
|
||||
break;
|
||||
case ::mediapipe::StatusCode::kUnimplemented:
|
||||
type = "Unimplemented";
|
||||
break;
|
||||
case ::mediapipe::StatusCode::kInternal:
|
||||
type = "Internal";
|
||||
break;
|
||||
case ::mediapipe::StatusCode::kUnavailable:
|
||||
type = "Unavailable";
|
||||
break;
|
||||
case ::mediapipe::StatusCode::kDataLoss:
|
||||
type = "Data loss";
|
||||
break;
|
||||
default:
|
||||
snprintf(tmp, sizeof(tmp), "Unknown code(%d)",
|
||||
static_cast<int>(code()));
|
||||
type = tmp;
|
||||
break;
|
||||
}
|
||||
std::string result(type);
|
||||
result += ": ";
|
||||
result += state_->msg;
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
void Status::IgnoreError() const {
|
||||
// no-op
|
||||
}
|
||||
|
||||
std::ostream& operator<<(std::ostream& os, const Status& x) {
|
||||
os << x.ToString();
|
||||
return os;
|
||||
|
||||
@@ -20,126 +20,16 @@
|
||||
#include <memory>
|
||||
#include <string>
|
||||
|
||||
#include "absl/status/status.h"
|
||||
#include "absl/strings/string_view.h"
|
||||
#include "mediapipe/framework/port/logging.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
enum class StatusCode {
|
||||
kOk = 0,
|
||||
kCancelled = 1,
|
||||
kUnknown = 2,
|
||||
kInvalidArgument = 3,
|
||||
kDeadlineExceeded = 4,
|
||||
kNotFound = 5,
|
||||
kAlreadyExists = 6,
|
||||
kPermissionDenied = 7,
|
||||
kResourceExhausted = 8,
|
||||
kFailedPrecondition = 9,
|
||||
kAborted = 10,
|
||||
kOutOfRange = 11,
|
||||
kUnimplemented = 12,
|
||||
kInternal = 13,
|
||||
kUnavailable = 14,
|
||||
kDataLoss = 15,
|
||||
kUnauthenticated = 16,
|
||||
kDoNotUseReservedForFutureExpansionUseDefaultInSwitchInstead_ = 20
|
||||
};
|
||||
using Status = absl::Status;
|
||||
using StatusCode = absl::StatusCode;
|
||||
|
||||
#if defined(__clang__)
|
||||
// Only clang supports warn_unused_result as a type annotation.
|
||||
class ABSL_MUST_USE_RESULT Status;
|
||||
#endif
|
||||
|
||||
// Denotes success or failure of a call in MediaPipe.
|
||||
class Status {
|
||||
public:
|
||||
// Creates a success status.
|
||||
Status() {}
|
||||
|
||||
// Creates a status with the specified error code and msg as a
|
||||
// human-readable std::string containing more detailed information.
|
||||
Status(::mediapipe::StatusCode code, absl::string_view msg);
|
||||
|
||||
// Copies the specified status.
|
||||
Status(const Status& s);
|
||||
void operator=(const Status& s);
|
||||
|
||||
// Returns true iff the status indicates success.
|
||||
bool ok() const {
|
||||
return (state_ == NULL) || (state_->code == ::mediapipe::StatusCode::kOk);
|
||||
}
|
||||
|
||||
::mediapipe::StatusCode code() const {
|
||||
return ok() ? ::mediapipe::StatusCode::kOk : state_->code;
|
||||
}
|
||||
|
||||
const std::string& error_message() const {
|
||||
return ok() ? empty_string() : state_->msg;
|
||||
}
|
||||
|
||||
absl::string_view message() const {
|
||||
return absl::string_view(error_message());
|
||||
}
|
||||
|
||||
bool operator==(const Status& x) const;
|
||||
bool operator!=(const Status& x) const;
|
||||
|
||||
// If `ok()`, stores `new_status` into `*this`. If `!ok()`,
|
||||
// preserves the current status, but may augment with additional
|
||||
// information about `new_status`.
|
||||
//
|
||||
// Convenient way of keeping track of the first error encountered.
|
||||
// Instead of:
|
||||
// `if (overall_status.ok()) overall_status = new_status`
|
||||
// Use:
|
||||
// `overall_status.Update(new_status);`
|
||||
void Update(const Status& new_status);
|
||||
|
||||
// Returns a std::string representation of this status suitable for
|
||||
// printing. Returns the std::string `"OK"` for success.
|
||||
std::string ToString() const;
|
||||
|
||||
// Ignores any errors. This method does nothing except potentially suppress
|
||||
// complaints from any tools that are checking that errors are not dropped on
|
||||
// the floor.
|
||||
void IgnoreError() const;
|
||||
|
||||
private:
|
||||
static const std::string& empty_string();
|
||||
struct State {
|
||||
::mediapipe::StatusCode code;
|
||||
std::string msg;
|
||||
};
|
||||
// OK status has a `NULL` state_. Otherwise, `state_` points to
|
||||
// a `State` structure containing the error code and message(s)
|
||||
std::unique_ptr<State> state_;
|
||||
|
||||
void SlowCopyFrom(const State* src);
|
||||
};
|
||||
|
||||
inline Status::Status(const Status& s)
|
||||
: state_((s.state_ == NULL) ? NULL : new State(*s.state_)) {}
|
||||
|
||||
inline void Status::operator=(const Status& s) {
|
||||
// The following condition catches both aliasing (when this == &s),
|
||||
// and the common case where both s and *this are ok.
|
||||
if (state_ != s.state_) {
|
||||
SlowCopyFrom(s.state_.get());
|
||||
}
|
||||
}
|
||||
|
||||
inline bool Status::operator==(const Status& x) const {
|
||||
return (this->state_ == x.state_) || (ToString() == x.ToString());
|
||||
}
|
||||
|
||||
inline bool Status::operator!=(const Status& x) const { return !(*this == x); }
|
||||
|
||||
inline Status OkStatus() { return Status(); }
|
||||
|
||||
std::ostream& operator<<(std::ostream& os, const Status& x);
|
||||
|
||||
typedef std::function<void(const Status&)> StatusCallback;
|
||||
inline ::mediapipe::Status OkStatus() { return absl::OkStatus(); }
|
||||
|
||||
extern std::string* MediaPipeCheckOpHelperOutOfLine(
|
||||
const ::mediapipe::Status& v, const char* msg);
|
||||
|
||||
@@ -72,12 +72,12 @@ StatusBuilder::operator Status() && {
|
||||
std::string message;
|
||||
if (join_style_ == MessageJoinStyle::kAnnotate) {
|
||||
if (!status_.ok()) {
|
||||
message = absl::StrCat(status_.error_message(), "; ", stream_->str());
|
||||
message = absl::StrCat(status_.message(), "; ", stream_->str());
|
||||
}
|
||||
} else {
|
||||
message = join_style_ == MessageJoinStyle::kPrepend
|
||||
? absl::StrCat(stream_->str(), status_.error_message())
|
||||
: absl::StrCat(status_.error_message(), stream_->str());
|
||||
? absl::StrCat(stream_->str(), status_.message())
|
||||
: absl::StrCat(status_.message(), stream_->str());
|
||||
}
|
||||
return Status(status_.code(), message);
|
||||
}
|
||||
|
||||
@@ -27,7 +27,7 @@ TEST(StatusBuilder, AnnotateMode) {
|
||||
<< "annotated message2";
|
||||
ASSERT_FALSE(status.ok());
|
||||
EXPECT_EQ(status.code(), ::mediapipe::StatusCode::kNotFound);
|
||||
EXPECT_EQ(status.error_message(),
|
||||
EXPECT_EQ(status.message(),
|
||||
"original message; annotated message1 annotated message2");
|
||||
}
|
||||
|
||||
@@ -42,7 +42,7 @@ TEST(StatusBuilder, PrependMode) {
|
||||
<< "prepended message2 ";
|
||||
ASSERT_FALSE(status.ok());
|
||||
EXPECT_EQ(status.code(), ::mediapipe::StatusCode::kInvalidArgument);
|
||||
EXPECT_EQ(status.error_message(),
|
||||
EXPECT_EQ(status.message(),
|
||||
"prepended message1 prepended message2 original message");
|
||||
}
|
||||
|
||||
@@ -56,8 +56,7 @@ TEST(StatusBuilder, AppendMode) {
|
||||
<< " extra message2";
|
||||
ASSERT_FALSE(status.ok());
|
||||
EXPECT_EQ(status.code(), ::mediapipe::StatusCode::kInternal);
|
||||
EXPECT_EQ(status.error_message(),
|
||||
"original message extra message1 extra message2");
|
||||
EXPECT_EQ(status.message(), "original message extra message1 extra message2");
|
||||
}
|
||||
|
||||
TEST(StatusBuilder, NoLoggingMode) {
|
||||
@@ -69,7 +68,7 @@ TEST(StatusBuilder, NoLoggingMode) {
|
||||
<< " extra message";
|
||||
ASSERT_FALSE(status.ok());
|
||||
EXPECT_EQ(status.code(), ::mediapipe::StatusCode::kUnavailable);
|
||||
EXPECT_EQ(status.error_message(), "original message");
|
||||
EXPECT_EQ(status.message(), "original message");
|
||||
}
|
||||
|
||||
} // namespace mediapipe
|
||||
|
||||