talksik/mediapipe
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Project import generated by Copybara.

Browse Source 
GitOrigin-RevId: 5aca6b3f07b67e09988a901f50f595ca5f566e67
This commit is contained in:
MediaPipe Team
2019-11-15 13:10:50 -08:00
committed by jqtang
parent d030c13931
commit 9437483827
116 changed files with 6284 additions and 391 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mediapipe/calculators/core/BUILD
+122 -3
View File
@@ -13,12 +13,12 @@
# limitations under the License.
#
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
licenses(["notice"]) # Apache 2.0
package(default_visibility = ["//visibility:private"])
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
proto_library(
name = "concatenate_vector_calculator_proto",
srcs = ["concatenate_vector_calculator.proto"],
@@ -79,6 +79,13 @@ proto_library(
],
)
proto_library(
name = "clip_vector_size_calculator_proto",
srcs = ["clip_vector_size_calculator.proto"],
visibility = ["//visibility:public"],
deps = ["//mediapipe/framework:calculator_proto"],
)
mediapipe_cc_proto_library(
name = "packet_cloner_calculator_cc_proto",
srcs = ["packet_cloner_calculator.proto"],
@@ -111,6 +118,14 @@ mediapipe_cc_proto_library(
deps = [":concatenate_vector_calculator_proto"],
)
mediapipe_cc_proto_library(
name = "clip_vector_size_calculator_cc_proto",
srcs = ["clip_vector_size_calculator.proto"],
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
visibility = ["//visibility:public"],
deps = [":clip_vector_size_calculator_proto"],
)
mediapipe_cc_proto_library(
name = "dequantize_byte_array_calculator_cc_proto",
srcs = ["dequantize_byte_array_calculator.proto"],
@@ -169,6 +184,66 @@ cc_test(
],
)
cc_library(
name = "begin_loop_calculator",
srcs = ["begin_loop_calculator.cc"],
hdrs = ["begin_loop_calculator.h"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_context",
"//mediapipe/framework:calculator_contract",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:collection_item_id",
"//mediapipe/framework:packet",
"//mediapipe/framework/formats:landmark_cc_proto",
"//mediapipe/framework/formats:rect_cc_proto",
"//mediapipe/framework/port:integral_types",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"@com_google_absl//absl/memory",
],
alwayslink = 1,
)
cc_library(
name = "end_loop_calculator",
srcs = ["end_loop_calculator.cc"],
hdrs = ["end_loop_calculator.h"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_context",
"//mediapipe/framework:calculator_contract",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:collection_item_id",
"//mediapipe/framework:packet",
"//mediapipe/framework/formats:landmark_cc_proto",
"//mediapipe/framework/formats:rect_cc_proto",
"//mediapipe/framework/port:integral_types",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"//mediapipe/util:render_data_cc_proto",
],
alwayslink = 1,
)
cc_test(
name = "begin_end_loop_calculator_graph_test",
srcs = ["begin_end_loop_calculator_graph_test.cc"],
deps = [
":begin_loop_calculator",
":end_loop_calculator",
"//mediapipe/calculators/core:packet_cloner_calculator",
"//mediapipe/framework:calculator_context",
"//mediapipe/framework:calculator_contract",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:integral_types",
"//mediapipe/framework/port:parse_text_proto",
"//mediapipe/framework/port:status",
"@com_google_absl//absl/memory",
],
)
cc_library(
name = "concatenate_vector_calculator",
srcs = ["concatenate_vector_calculator.cc"],
@@ -219,6 +294,50 @@ cc_test(
],
)
cc_library(
name = "clip_vector_size_calculator",
srcs = ["clip_vector_size_calculator.cc"],
hdrs = ["clip_vector_size_calculator.h"],
visibility = ["//visibility:public"],
deps = [
":clip_vector_size_calculator_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/formats:rect_cc_proto",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"@org_tensorflow//tensorflow/lite:framework",
],
alwayslink = 1,
)
cc_library(
name = "clip_detection_vector_size_calculator",
srcs = ["clip_detection_vector_size_calculator.cc"],
visibility = ["//visibility:public"],
deps = [
":clip_vector_size_calculator",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/formats:detection_cc_proto",
],
alwayslink = 1,
)
cc_test(
name = "clip_vector_size_calculator_test",
srcs = ["clip_vector_size_calculator_test.cc"],
deps = [
":clip_vector_size_calculator",
"//mediapipe/calculators/core:packet_resampler_calculator_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:calculator_runner",
"//mediapipe/framework:timestamp",
"//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:parse_text_proto",
"//mediapipe/framework/port:status",
"@com_google_absl//absl/strings",
],
)
cc_library(
name = "counting_source_calculator",
srcs = ["counting_source_calculator.cc"],
@@ -300,7 +419,7 @@ cc_library(
"//visibility:public",
],
deps = [
"//mediapipe/calculators/core:packet_cloner_calculator_cc_proto",
":packet_cloner_calculator_cc_proto",
"//mediapipe/framework:calculator_framework",
"@com_google_absl//absl/strings",
],
mediapipe/calculators/core/begin_end_loop_calculator_graph_test.cc
+335
View File
@@ -0,0 +1,335 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <string>
#include <vector>
#include "absl/memory/memory.h"
#include "mediapipe/calculators/core/begin_loop_calculator.h"
#include "mediapipe/calculators/core/end_loop_calculator.h"
#include "mediapipe/framework/calculator_contract.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/port/gtest.h"
#include "mediapipe/framework/port/integral_types.h"
#include "mediapipe/framework/port/parse_text_proto.h"
#include "mediapipe/framework/port/status_matchers.h" // NOLINT
namespace mediapipe {
namespace {
typedef BeginLoopCalculator<std::vector<int>> BeginLoopIntegerCalculator;
REGISTER_CALCULATOR(BeginLoopIntegerCalculator);
class IncrementCalculator : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
cc->Inputs().Index(0).Set<int>();
cc->Outputs().Index(0).Set<int>();
return ::mediapipe::OkStatus();
}
::mediapipe::Status Open(CalculatorContext* cc) override {
cc->SetOffset(TimestampDiff(0));
return ::mediapipe::OkStatus();
}
::mediapipe::Status Process(CalculatorContext* cc) override {
const int& input_int = cc->Inputs().Index(0).Get<int>();
auto output_int = absl::make_unique<int>(input_int + 1);
cc->Outputs().Index(0).Add(output_int.release(), cc->InputTimestamp());
return ::mediapipe::OkStatus();
}
};
REGISTER_CALCULATOR(IncrementCalculator);
typedef EndLoopCalculator<std::vector<int>> EndLoopIntegersCalculator;
REGISTER_CALCULATOR(EndLoopIntegersCalculator);
class BeginEndLoopCalculatorGraphTest : public ::testing::Test {
protected:
BeginEndLoopCalculatorGraphTest() {
graph_config_ = ParseTextProtoOrDie<CalculatorGraphConfig>(
R"(
num_threads: 4
input_stream: "ints"
node {
calculator: "BeginLoopIntegerCalculator"
input_stream: "ITERABLE:ints"
output_stream: "ITEM:int"
output_stream: "BATCH_END:timestamp"
}
node {
calculator: "IncrementCalculator"
input_stream: "int"
output_stream: "int_plus_one"
}
node {
calculator: "EndLoopIntegersCalculator"
input_stream: "ITEM:int_plus_one"
input_stream: "BATCH_END:timestamp"
output_stream: "ITERABLE:ints_plus_one"
}
)");
tool::AddVectorSink("ints_plus_one", &graph_config_, &output_packets_);
}
CalculatorGraphConfig graph_config_;
std::vector<Packet> output_packets_;
};
TEST_F(BeginEndLoopCalculatorGraphTest, SingleEmptyVector) {
CalculatorGraph graph;
MP_EXPECT_OK(graph.Initialize(graph_config_));
MP_EXPECT_OK(graph.StartRun({}));
auto input_vector = absl::make_unique<std::vector<int>>();
Timestamp input_timestamp = Timestamp(0);
MP_ASSERT_OK(graph.AddPacketToInputStream(
"ints", Adopt(input_vector.release()).At(input_timestamp)));
MP_ASSERT_OK(graph.WaitUntilIdle());
// EndLoopCalc will forward the timestamp bound because there are no elements
// in collection to output.
ASSERT_EQ(0, output_packets_.size());
MP_ASSERT_OK(graph.CloseAllPacketSources());
MP_ASSERT_OK(graph.WaitUntilDone());
}
TEST_F(BeginEndLoopCalculatorGraphTest, SingleNonEmptyVector) {
CalculatorGraph graph;
MP_EXPECT_OK(graph.Initialize(graph_config_));
MP_EXPECT_OK(graph.StartRun({}));
auto input_vector = absl::make_unique<std::vector<int>>();
input_vector->emplace_back(0);
input_vector->emplace_back(1);
input_vector->emplace_back(2);
Timestamp input_timestamp = Timestamp(0);
MP_ASSERT_OK(graph.AddPacketToInputStream(
"ints", Adopt(input_vector.release()).At(input_timestamp)));
MP_ASSERT_OK(graph.WaitUntilIdle());
ASSERT_EQ(1, output_packets_.size());
EXPECT_EQ(input_timestamp, output_packets_[0].Timestamp());
std::vector<int> expected_output_vector = {1, 2, 3};
EXPECT_EQ(expected_output_vector, output_packets_[0].Get<std::vector<int>>());
MP_ASSERT_OK(graph.CloseAllPacketSources());
MP_ASSERT_OK(graph.WaitUntilDone());
}
TEST_F(BeginEndLoopCalculatorGraphTest, MultipleVectors) {
CalculatorGraph graph;
MP_EXPECT_OK(graph.Initialize(graph_config_));
MP_EXPECT_OK(graph.StartRun({}));
auto input_vector0 = absl::make_unique<std::vector<int>>();
input_vector0->emplace_back(0);
input_vector0->emplace_back(1);
Timestamp input_timestamp0 = Timestamp(0);
MP_ASSERT_OK(graph.AddPacketToInputStream(
"ints", Adopt(input_vector0.release()).At(input_timestamp0)));
auto input_vector1 = absl::make_unique<std::vector<int>>();
Timestamp input_timestamp1 = Timestamp(1);
MP_ASSERT_OK(graph.AddPacketToInputStream(
"ints", Adopt(input_vector1.release()).At(input_timestamp1)));
auto input_vector2 = absl::make_unique<std::vector<int>>();
input_vector2->emplace_back(2);
input_vector2->emplace_back(3);
Timestamp input_timestamp2 = Timestamp(2);
MP_ASSERT_OK(graph.AddPacketToInputStream(
"ints", Adopt(input_vector2.release()).At(input_timestamp2)));
MP_ASSERT_OK(graph.CloseAllPacketSources());
MP_ASSERT_OK(graph.WaitUntilDone());
ASSERT_EQ(2, output_packets_.size());
EXPECT_EQ(input_timestamp0, output_packets_[0].Timestamp());
std::vector<int> expected_output_vector0 = {1, 2};
EXPECT_EQ(expected_output_vector0,
output_packets_[0].Get<std::vector<int>>());
// At input_timestamp1, EndLoopCalc will forward timestamp bound as there are
// no elements in vector to process.
EXPECT_EQ(input_timestamp2, output_packets_[1].Timestamp());
std::vector<int> expected_output_vector2 = {3, 4};
EXPECT_EQ(expected_output_vector2,
output_packets_[1].Get<std::vector<int>>());
}
class MultiplierCalculator : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
cc->Inputs().Index(0).Set<int>();
cc->Inputs().Index(1).Set<int>();
cc->Outputs().Index(0).Set<int>();
return ::mediapipe::OkStatus();
}
::mediapipe::Status Open(CalculatorContext* cc) override {
cc->SetOffset(TimestampDiff(0));
return ::mediapipe::OkStatus();
}
::mediapipe::Status Process(CalculatorContext* cc) override {
const int& input_int = cc->Inputs().Index(0).Get<int>();
const int& multiplier_int = cc->Inputs().Index(1).Get<int>();
auto output_int = absl::make_unique<int>(input_int * multiplier_int);
cc->Outputs().Index(0).Add(output_int.release(), cc->InputTimestamp());
return ::mediapipe::OkStatus();
}
};
REGISTER_CALCULATOR(MultiplierCalculator);
class BeginEndLoopCalculatorGraphWithClonedInputsTest : public ::testing::Test {
protected:
BeginEndLoopCalculatorGraphWithClonedInputsTest() {
graph_config_ = ParseTextProtoOrDie<CalculatorGraphConfig>(
R"(
num_threads: 4
input_stream: "ints"
input_stream: "multiplier"
node {
calculator: "BeginLoopIntegerCalculator"
input_stream: "ITERABLE:ints"
input_stream: "CLONE:multiplier"
output_stream: "ITEM:int_at_loop"
output_stream: "CLONE:multiplier_cloned_at_loop"
output_stream: "BATCH_END:timestamp"
}
node {
calculator: "MultiplierCalculator"
input_stream: "int_at_loop"
input_stream: "multiplier_cloned_at_loop"
output_stream: "multiplied_int_at_loop"
}
node {
calculator: "EndLoopIntegersCalculator"
input_stream: "ITEM:multiplied_int_at_loop"
input_stream: "BATCH_END:timestamp"
output_stream: "ITERABLE:multiplied_ints"
}
)");
tool::AddVectorSink("multiplied_ints", &graph_config_, &output_packets_);
}
CalculatorGraphConfig graph_config_;
std::vector<Packet> output_packets_;
};
TEST_F(BeginEndLoopCalculatorGraphWithClonedInputsTest, SingleEmptyVector) {
CalculatorGraph graph;
MP_EXPECT_OK(graph.Initialize(graph_config_));
MP_EXPECT_OK(graph.StartRun({}));
auto input_vector = absl::make_unique<std::vector<int>>();
Timestamp input_timestamp = Timestamp(42);
MP_ASSERT_OK(graph.AddPacketToInputStream(
"ints", Adopt(input_vector.release()).At(input_timestamp)));
auto multiplier = absl::make_unique<int>(2);
MP_ASSERT_OK(graph.AddPacketToInputStream(
"multiplier", Adopt(multiplier.release()).At(input_timestamp)));
MP_ASSERT_OK(graph.WaitUntilIdle());
// EndLoopCalc will forward the timestamp bound because there are no elements
// in collection to output.
ASSERT_EQ(0, output_packets_.size());
MP_ASSERT_OK(graph.CloseAllPacketSources());
MP_ASSERT_OK(graph.WaitUntilDone());
}
TEST_F(BeginEndLoopCalculatorGraphWithClonedInputsTest, SingleNonEmptyVector) {
CalculatorGraph graph;
MP_EXPECT_OK(graph.Initialize(graph_config_));
MP_EXPECT_OK(graph.StartRun({}));
auto input_vector = absl::make_unique<std::vector<int>>();
input_vector->emplace_back(0);
input_vector->emplace_back(1);
input_vector->emplace_back(2);
Timestamp input_timestamp = Timestamp(42);
MP_ASSERT_OK(graph.AddPacketToInputStream(
"ints", Adopt(input_vector.release()).At(input_timestamp)));
auto multiplier = absl::make_unique<int>(2);
MP_ASSERT_OK(graph.AddPacketToInputStream(
"multiplier", Adopt(multiplier.release()).At(input_timestamp)));
MP_ASSERT_OK(graph.WaitUntilIdle());
ASSERT_EQ(1, output_packets_.size());
EXPECT_EQ(input_timestamp, output_packets_[0].Timestamp());
std::vector<int> expected_output_vector = {0, 2, 4};
EXPECT_EQ(expected_output_vector, output_packets_[0].Get<std::vector<int>>());
MP_ASSERT_OK(graph.CloseAllPacketSources());
MP_ASSERT_OK(graph.WaitUntilDone());
}
TEST_F(BeginEndLoopCalculatorGraphWithClonedInputsTest, MultipleVectors) {
CalculatorGraph graph;
MP_EXPECT_OK(graph.Initialize(graph_config_));
MP_EXPECT_OK(graph.StartRun({}));
auto input_vector0 = absl::make_unique<std::vector<int>>();
input_vector0->emplace_back(0);
input_vector0->emplace_back(1);
Timestamp input_timestamp0 = Timestamp(42);
MP_ASSERT_OK(graph.AddPacketToInputStream(
"ints", Adopt(input_vector0.release()).At(input_timestamp0)));
auto multiplier0 = absl::make_unique<int>(2);
MP_ASSERT_OK(graph.AddPacketToInputStream(
"multiplier", Adopt(multiplier0.release()).At(input_timestamp0)));
auto input_vector1 = absl::make_unique<std::vector<int>>();
Timestamp input_timestamp1 = Timestamp(43);
MP_ASSERT_OK(graph.AddPacketToInputStream(
"ints", Adopt(input_vector1.release()).At(input_timestamp1)));
auto multiplier1 = absl::make_unique<int>(2);
MP_ASSERT_OK(graph.AddPacketToInputStream(
"multiplier", Adopt(multiplier1.release()).At(input_timestamp1)));
auto input_vector2 = absl::make_unique<std::vector<int>>();
input_vector2->emplace_back(2);
input_vector2->emplace_back(3);
Timestamp input_timestamp2 = Timestamp(44);
MP_ASSERT_OK(graph.AddPacketToInputStream(
"ints", Adopt(input_vector2.release()).At(input_timestamp2)));
auto multiplier2 = absl::make_unique<int>(3);
MP_ASSERT_OK(graph.AddPacketToInputStream(
"multiplier", Adopt(multiplier2.release()).At(input_timestamp2)));
MP_ASSERT_OK(graph.CloseAllPacketSources());
MP_ASSERT_OK(graph.WaitUntilDone());
ASSERT_EQ(2, output_packets_.size());
EXPECT_EQ(input_timestamp0, output_packets_[0].Timestamp());
std::vector<int> expected_output_vector0 = {0, 2};
EXPECT_EQ(expected_output_vector0,
output_packets_[0].Get<std::vector<int>>());
// At input_timestamp1, EndLoopCalc will forward timestamp bound as there are
// no elements in vector to process.
EXPECT_EQ(input_timestamp2, output_packets_[1].Timestamp());
std::vector<int> expected_output_vector2 = {6, 9};
EXPECT_EQ(expected_output_vector2,
output_packets_[1].Get<std::vector<int>>());
}
} // namespace
} // namespace mediapipe
mediapipe/calculators/core/begin_loop_calculator.cc
+40
View File
@@ -0,0 +1,40 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/calculators/core/begin_loop_calculator.h"
#include <vector>
#include "mediapipe/framework/formats/landmark.pb.h"
#include "mediapipe/framework/formats/rect.pb.h"
namespace mediapipe {
// A calculator to process std::vector<NormalizedLandmark>.
typedef BeginLoopCalculator<std::vector<::mediapipe::NormalizedLandmark>>
BeginLoopNormalizedLandmarkCalculator;
REGISTER_CALCULATOR(BeginLoopNormalizedLandmarkCalculator);
// A calculator to process std::vector<std::vector<NormalizedLandmark>>.
typedef BeginLoopCalculator<
std::vector<std::vector<::mediapipe::NormalizedLandmark>>>
BeginLoopNormalizedLandmarksVectorCalculator;
REGISTER_CALCULATOR(BeginLoopNormalizedLandmarksVectorCalculator);
// A calculator to process std::vector<NormalizedRect>.
typedef BeginLoopCalculator<std::vector<::mediapipe::NormalizedRect>>
BeginLoopNormalizedRectCalculator;
REGISTER_CALCULATOR(BeginLoopNormalizedRectCalculator);
} // namespace mediapipe
mediapipe/calculators/core/begin_loop_calculator.h
+157
View File
@@ -0,0 +1,157 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef MEDIAPIPE_CALCULATORS_CORE_BEGIN_LOOP_CALCULATOR_H_
#define MEDIAPIPE_CALCULATORS_CORE_BEGIN_LOOP_CALCULATOR_H_
#include "absl/memory/memory.h"
#include "mediapipe/framework/calculator_context.h"
#include "mediapipe/framework/calculator_contract.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/collection_item_id.h"
#include "mediapipe/framework/packet.h"
#include "mediapipe/framework/port/integral_types.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status.h"
namespace mediapipe {
// Calculator for implementing loops on iterable collections inside a MediaPipe
// graph.
//
// It is designed to be used like:
//
// node {
// calculator: "BeginLoopWithIterableCalculator"
// input_stream: "ITERABLE:input_iterable" # IterableT @ext_ts
// output_stream: "ITEM:input_element" # ItemT @loop_internal_ts
// output_stream: "BATCH_END:ext_ts" # Timestamp @loop_internal_ts
// }
//
// node {
// calculator: "ElementToBlaConverterSubgraph"
// input_stream: "ITEM:input_to_loop_body" # ItemT @loop_internal_ts
// output_stream: "BLA:output_of_loop_body" # ItemU @loop_internal_ts
// }
//
// node {
// calculator: "EndLoopWithOutputCalculator"
// input_stream: "ITEM:output_of_loop_body" # ItemU @loop_internal_ts
// input_stream: "BATCH_END:ext_ts" # Timestamp @loop_internal_ts
// output_stream: "OUTPUT:aggregated_result" # IterableU @ext_ts
// }
//
// BeginLoopCalculator accepts an optional input stream tagged with "TICK"
// which if non-empty, wakes up the calculator and calls
// BeginLoopCalculator::Process(). Input streams tagged with "CLONE" are cloned
// to the corresponding output streams at loop timestamps. This ensures that a
// MediaPipe graph or sub-graph can run multiple times, once per element in the
// "ITERABLE" for each pakcet clone of the packets in the "CLONE" input streams.
template <typename IterableT>
class BeginLoopCalculator : public CalculatorBase {
using ItemT = typename IterableT::value_type;
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
// A non-empty packet in the optional "TICK" input stream wakes up the
// calculator.
if (cc->Inputs().HasTag("TICK")) {
cc->Inputs().Tag("TICK").SetAny();
}
// An iterable collection in the input stream.
RET_CHECK(cc->Inputs().HasTag("ITERABLE"));
cc->Inputs().Tag("ITERABLE").Set<IterableT>();
// An element from the collection.
RET_CHECK(cc->Outputs().HasTag("ITEM"));
cc->Outputs().Tag("ITEM").Set<ItemT>();
RET_CHECK(cc->Outputs().HasTag("BATCH_END"));
cc->Outputs()
.Tag("BATCH_END")
.Set<Timestamp>(
// A flush signal to the corresponding EndLoopCalculator for it to
// emit the aggregated result with the timestamp contained in this
// flush signal packet.
);
// Input streams tagged with "CLONE" are cloned to the corresponding
// "CLONE" output streams at loop timestamps.
RET_CHECK(cc->Inputs().NumEntries("CLONE") ==
cc->Outputs().NumEntries("CLONE"));
if (cc->Inputs().NumEntries("CLONE") > 0) {
for (int i = 0; i < cc->Inputs().NumEntries("CLONE"); ++i) {
cc->Inputs().Get("CLONE", i).SetAny();
cc->Outputs().Get("CLONE", i).SetSameAs(&cc->Inputs().Get("CLONE", i));
}
}
return ::mediapipe::OkStatus();
}
::mediapipe::Status Process(CalculatorContext* cc) final {
Timestamp last_timestamp = loop_internal_timestamp_;
if (!cc->Inputs().Tag("ITERABLE").IsEmpty()) {
const IterableT& collection =
cc->Inputs().Tag("ITERABLE").template Get<IterableT>();
for (const auto& item : collection) {
cc->Outputs().Tag("ITEM").AddPacket(
MakePacket<ItemT>(item).At(loop_internal_timestamp_));
ForwardClonePackets(cc, loop_internal_timestamp_);
++loop_internal_timestamp_;
}
}
// The collection was empty and nothing was processed.
if (last_timestamp == loop_internal_timestamp_) {
// Increment loop_internal_timestamp_ because it is used up now.
++loop_internal_timestamp_;
for (auto it = cc->Outputs().begin(); it < cc->Outputs().end(); ++it) {
it->SetNextTimestampBound(loop_internal_timestamp_);
}
}
// The for loop processing the input collection already incremented
// loop_internal_timestamp_. To emit BATCH_END packet along the last
// non-BATCH_END packet, decrement by one.
cc->Outputs()
.Tag("BATCH_END")
.AddPacket(MakePacket<Timestamp>(cc->InputTimestamp())
.At(Timestamp(loop_internal_timestamp_ - 1)));
return ::mediapipe::OkStatus();
}
private:
void ForwardClonePackets(CalculatorContext* cc, Timestamp output_timestamp) {
if (cc->Inputs().NumEntries("CLONE") > 0) {
for (int i = 0; i < cc->Inputs().NumEntries("CLONE"); ++i) {
if (!cc->Inputs().Get("CLONE", i).IsEmpty()) {
auto input_packet = cc->Inputs().Get("CLONE", i).Value();
cc->Outputs()
.Get("CLONE", i)
.AddPacket(std::move(input_packet).At(output_timestamp));
}
}
}
}
// Fake timestamps generated per element in collection.
Timestamp loop_internal_timestamp_ = Timestamp(0);
};
} // namespace mediapipe
#endif // MEDIAPIPE_CALCULATORS_CORE_BEGIN_LOOP_CALCULATOR_H_
mediapipe/calculators/core/clip_detection_vector_size_calculator.cc
+26
View File
@@ -0,0 +1,26 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <vector>
#include "mediapipe/calculators/core/clip_vector_size_calculator.h"
#include "mediapipe/framework/formats/detection.pb.h"
namespace mediapipe {
typedef ClipVectorSizeCalculator<::mediapipe::Detection>
ClipDetectionVectorSizeCalculator;
REGISTER_CALCULATOR(ClipDetectionVectorSizeCalculator);
} // namespace mediapipe
mediapipe/calculators/core/clip_vector_size_calculator.cc
+28
View File
@@ -0,0 +1,28 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/calculators/core/clip_vector_size_calculator.h"
#include <vector>
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/rect.pb.h"
namespace mediapipe {
typedef ClipVectorSizeCalculator<::mediapipe::NormalizedRect>
ClipNormalizedRectVectorSizeCalculator;
REGISTER_CALCULATOR(ClipNormalizedRectVectorSizeCalculator);
} // namespace mediapipe
mediapipe/calculators/core/clip_vector_size_calculator.h
+137
View File
@@ -0,0 +1,137 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef MEDIAPIPE_CALCULATORS_CORE_CLIP_VECTOR_SIZE_CALCULATOR_H_
#define MEDIAPIPE_CALCULATORS_CORE_CLIP_VECTOR_SIZE_CALCULATOR_H_
#include <type_traits>
#include <vector>
#include "mediapipe/calculators/core/clip_vector_size_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/port/canonical_errors.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status.h"
namespace mediapipe {
// Clips the size of the input vector of type T to a specified max_vec_size.
// In a graph it will be used as:
// node {
// calculator: "ClipIntVectorSizeCalculator"
// input_stream: "input_vector"
// output_stream: "output_vector"
// options {
// [mediapipe.ClipIntVectorSizeCalculatorOptions.ext] {
// max_vec_size: 5
// }
// }
// }
template <typename T>
class ClipVectorSizeCalculator : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
RET_CHECK(cc->Inputs().NumEntries() == 1);
RET_CHECK(cc->Outputs().NumEntries() == 1);
if (cc->Options<::mediapipe::ClipVectorSizeCalculatorOptions>()
.max_vec_size() < 1) {
return ::mediapipe::InternalError(
"max_vec_size should be greater than or equal to 1.");
}
cc->Inputs().Index(0).Set<std::vector<T>>();
cc->Outputs().Index(0).Set<std::vector<T>>();
return ::mediapipe::OkStatus();
}
::mediapipe::Status Open(CalculatorContext* cc) override {
cc->SetOffset(TimestampDiff(0));
max_vec_size_ = cc->Options<::mediapipe::ClipVectorSizeCalculatorOptions>()
.max_vec_size();
return ::mediapipe::OkStatus();
}
::mediapipe::Status Process(CalculatorContext* cc) override {
if (max_vec_size_ < 1) {
return ::mediapipe::InternalError(
"max_vec_size should be greater than or equal to 1.");
}
if (cc->Inputs().Index(0).IsEmpty()) {
return ::mediapipe::OkStatus();
}
return ClipVectorSize<T>(std::is_copy_constructible<T>(), cc);
}
template <typename U>
::mediapipe::Status ClipVectorSize(std::true_type, CalculatorContext* cc) {
auto output = absl::make_unique<std::vector<U>>();
const std::vector<U>& input_vector =
cc->Inputs().Index(0).Get<std::vector<U>>();
if (max_vec_size_ >= input_vector.size()) {
output->insert(output->end(), input_vector.begin(), input_vector.end());
} else {
for (int i = 0; i < max_vec_size_; ++i) {
output->push_back(input_vector[i]);
}
}
cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp());
return ::mediapipe::OkStatus();
}
template <typename U>
::mediapipe::Status ClipVectorSize(std::false_type, CalculatorContext* cc) {
return ConsumeAndClipVectorSize<T>(std::is_move_constructible<U>(), cc);
}
template <typename U>
::mediapipe::Status ConsumeAndClipVectorSize(std::true_type,
CalculatorContext* cc) {
auto output = absl::make_unique<std::vector<U>>();
::mediapipe::StatusOr<std::unique_ptr<std::vector<U>>> input_status =
cc->Inputs().Index(0).Value().Consume<std::vector<U>>();
if (input_status.ok()) {
std::unique_ptr<std::vector<U>> input_vector =
std::move(input_status).ValueOrDie();
auto begin_it = input_vector->begin();
auto end_it = input_vector->end();
if (max_vec_size_ < input_vector->size()) {
end_it = input_vector->begin() + max_vec_size_;
}
output->insert(output->end(), std::make_move_iterator(begin_it),
std::make_move_iterator(end_it));
} else {
return input_status.status();
}
cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp());
return ::mediapipe::OkStatus();
}
template <typename U>
::mediapipe::Status ConsumeAndClipVectorSize(std::false_type,
CalculatorContext* cc) {
return ::mediapipe::InternalError(
"Cannot copy or move input vectors and clip their size.");
}
private:
int max_vec_size_ = 0;
};
} // namespace mediapipe
#endif // MEDIAPIPE_CALCULATORS_CORE_CLIP_VECTOR_SIZE_CALCULATOR_H_
mediapipe/calculators/core/clip_vector_size_calculator.proto
+28
View File
@@ -0,0 +1,28 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto2";
package mediapipe;
import "mediapipe/framework/calculator.proto";
message ClipVectorSizeCalculatorOptions {
extend CalculatorOptions {
optional ClipVectorSizeCalculatorOptions ext = 274674998;
}
// Maximum size of output vector.
optional int32 max_vec_size = 1 [default = 1];
}
mediapipe/calculators/core/clip_vector_size_calculator_test.cc
+179
View File
@@ -0,0 +1,179 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/calculators/core/clip_vector_size_calculator.h"
#include <memory>
#include <string>
#include <vector>
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_runner.h"
#include "mediapipe/framework/port/gmock.h"
#include "mediapipe/framework/port/gtest.h"
#include "mediapipe/framework/port/parse_text_proto.h"
#include "mediapipe/framework/port/status_matchers.h" // NOLINT
namespace mediapipe {
typedef ClipVectorSizeCalculator<int> TestClipIntVectorSizeCalculator;
REGISTER_CALCULATOR(TestClipIntVectorSizeCalculator);
void AddInputVector(const std::vector<int>& input, int64 timestamp,
CalculatorRunner* runner) {
runner->MutableInputs()->Index(0).packets.push_back(
MakePacket<std::vector<int>>(input).At(Timestamp(timestamp)));
}
TEST(TestClipIntVectorSizeCalculatorTest, EmptyVectorInput) {
CalculatorGraphConfig::Node node_config =
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
calculator: "TestClipIntVectorSizeCalculator"
input_stream: "input_vector"
output_stream: "output_vector"
options {
[mediapipe.ClipVectorSizeCalculatorOptions.ext] { max_vec_size: 1 }
}
)");
CalculatorRunner runner(node_config);
std::vector<int> input = {};
AddInputVector(input, /*timestamp=*/1, &runner);
MP_ASSERT_OK(runner.Run());
const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
EXPECT_EQ(1, outputs.size());
EXPECT_EQ(Timestamp(1), outputs[0].Timestamp());
EXPECT_TRUE(outputs[0].Get<std::vector<int>>().empty());
}
TEST(TestClipIntVectorSizeCalculatorTest, OneTimestamp) {
CalculatorGraphConfig::Node node_config =
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
calculator: "TestClipIntVectorSizeCalculator"
input_stream: "input_vector"
output_stream: "output_vector"
options {
[mediapipe.ClipVectorSizeCalculatorOptions.ext] { max_vec_size: 2 }
}
)");
CalculatorRunner runner(node_config);
std::vector<int> input = {0, 1, 2, 3};
AddInputVector(input, /*timestamp=*/1, &runner);
MP_ASSERT_OK(runner.Run());
const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
EXPECT_EQ(1, outputs.size());
EXPECT_EQ(Timestamp(1), outputs[0].Timestamp());
const std::vector<int>& output = outputs[0].Get<std::vector<int>>();
EXPECT_EQ(2, output.size());
std::vector<int> expected_vector = {0, 1};
EXPECT_EQ(expected_vector, output);
}
TEST(TestClipIntVectorSizeCalculatorTest, TwoInputsAtTwoTimestamps) {
CalculatorGraphConfig::Node node_config =
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
calculator: "TestClipIntVectorSizeCalculator"
input_stream: "input_vector"
output_stream: "output_vector"
options {
[mediapipe.ClipVectorSizeCalculatorOptions.ext] { max_vec_size: 3 }
}
)");
CalculatorRunner runner(node_config);
{
std::vector<int> input = {0, 1, 2, 3};
AddInputVector(input, /*timestamp=*/1, &runner);
}
{
std::vector<int> input = {2, 3, 4, 5};
AddInputVector(input, /*timestamp=*/2, &runner);
}
MP_ASSERT_OK(runner.Run());
const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
EXPECT_EQ(2, outputs.size());
{
EXPECT_EQ(Timestamp(1), outputs[0].Timestamp());
const std::vector<int>& output = outputs[0].Get<std::vector<int>>();
EXPECT_EQ(3, output.size());
std::vector<int> expected_vector = {0, 1, 2};
EXPECT_EQ(expected_vector, output);
}
{
EXPECT_EQ(Timestamp(2), outputs[1].Timestamp());
const std::vector<int>& output = outputs[1].Get<std::vector<int>>();
EXPECT_EQ(3, output.size());
std::vector<int> expected_vector = {2, 3, 4};
EXPECT_EQ(expected_vector, output);
}
}
typedef ClipVectorSizeCalculator<std::unique_ptr<int>>
TestClipUniqueIntPtrVectorSizeCalculator;
REGISTER_CALCULATOR(TestClipUniqueIntPtrVectorSizeCalculator);
TEST(TestClipUniqueIntPtrVectorSizeCalculatorTest, ConsumeOneTimestamp) {
/* Note: We don't use CalculatorRunner for this test because it keeps copies
* of input packets, so packets sent to the graph don't have sole ownership.
* The test needs to send packets that own the data.
*/
CalculatorGraphConfig graph_config =
ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
input_stream: "input_vector"
node {
calculator: "TestClipUniqueIntPtrVectorSizeCalculator"
input_stream: "input_vector"
output_stream: "output_vector"
options {
[mediapipe.ClipVectorSizeCalculatorOptions.ext] { max_vec_size: 3 }
}
}
)");
std::vector<Packet> outputs;
tool::AddVectorSink("output_vector", &graph_config, &outputs);
CalculatorGraph graph;
MP_EXPECT_OK(graph.Initialize(graph_config));
MP_EXPECT_OK(graph.StartRun({}));
// input1 : {0, 1, 2, 3, 4, 5}
auto input_vector = absl::make_unique<std::vector<std::unique_ptr<int>>>(6);
for (int i = 0; i < 6; ++i) {
input_vector->at(i) = absl::make_unique<int>(i);
}
MP_EXPECT_OK(graph.AddPacketToInputStream(
"input_vector", Adopt(input_vector.release()).At(Timestamp(1))));
MP_EXPECT_OK(graph.WaitUntilIdle());
MP_EXPECT_OK(graph.CloseAllPacketSources());
MP_EXPECT_OK(graph.WaitUntilDone());
EXPECT_EQ(1, outputs.size());
EXPECT_EQ(Timestamp(1), outputs[0].Timestamp());
const std::vector<std::unique_ptr<int>>& result =
outputs[0].Get<std::vector<std::unique_ptr<int>>>();
EXPECT_EQ(3, result.size());
for (int i = 0; i < 3; ++i) {
const std::unique_ptr<int>& v = result[i];
EXPECT_EQ(i, *v);
}
}
} // namespace mediapipe
mediapipe/calculators/core/end_loop_calculator.cc
+45
View File
@@ -0,0 +1,45 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/calculators/core/end_loop_calculator.h"
#include <vector>
#include "mediapipe/framework/formats/landmark.pb.h"
#include "mediapipe/framework/formats/rect.pb.h"
#include "mediapipe/util/render_data.pb.h"
namespace mediapipe {
typedef EndLoopCalculator<std::vector<::mediapipe::NormalizedRect>>
EndLoopNormalizedRectCalculator;
REGISTER_CALCULATOR(EndLoopNormalizedRectCalculator);
typedef EndLoopCalculator<std::vector<::mediapipe::NormalizedLandmark>>
EndLoopNormalizedLandmarkCalculator;
REGISTER_CALCULATOR(EndLoopNormalizedLandmarkCalculator);
typedef EndLoopCalculator<
std::vector<std::vector<::mediapipe::NormalizedLandmark>>>
EndLoopNormalizedLandmarksVectorCalculator;
REGISTER_CALCULATOR(EndLoopNormalizedLandmarksVectorCalculator);
typedef EndLoopCalculator<std::vector<bool>> EndLoopBooleanCalculator;
REGISTER_CALCULATOR(EndLoopBooleanCalculator);
typedef EndLoopCalculator<std::vector<::mediapipe::RenderData>>
EndLoopRenderDataCalculator;
REGISTER_CALCULATOR(EndLoopRenderDataCalculator);
} // namespace mediapipe
mediapipe/calculators/core/end_loop_calculator.h
+106
View File
@@ -0,0 +1,106 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef MEDIAPIPE_CALCULATORS_CORE_END_LOOP_CALCULATOR_H_
#define MEDIAPIPE_CALCULATORS_CORE_END_LOOP_CALCULATOR_H_
#include "mediapipe/framework/calculator_context.h"
#include "mediapipe/framework/calculator_contract.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/collection_item_id.h"
#include "mediapipe/framework/port/integral_types.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status.h"
namespace mediapipe {
// Calculator for completing the processing of loops on iterable collections
// inside a MediaPipe graph. The EndLoopCalculator collects all input packets
// from ITEM input_stream into a collection and upon receiving the flush signal
// from the "BATCH_END" tagged input stream, it emits the aggregated results
// at the original timestamp contained in the "BATCH_END" input stream.
//
// It is designed to be used like:
//
// node {
// calculator: "BeginLoopWithIterableCalculator"
// input_stream: "ITERABLE:input_iterable" # IterableT @ext_ts
// output_stream: "ITEM:input_element" # ItemT @loop_internal_ts
// output_stream: "BATCH_END:ext_ts" # Timestamp @loop_internal_ts
// }
//
// node {
// calculator: "ElementToBlaConverterSubgraph"
// input_stream: "ITEM:input_to_loop_body" # ItemT @loop_internal_ts
// output_stream: "BLA:output_of_loop_body" # ItemU @loop_internal_ts
// }
//
// node {
// calculator: "EndLoopWithOutputCalculator"
// input_stream: "ITEM:output_of_loop_body" # ItemU @loop_internal_ts
// input_stream: "BATCH_END:ext_ts" # Timestamp @loop_internal_ts
// output_stream: "OUTPUT:aggregated_result" # IterableU @ext_ts
// }
template <typename IterableT>
class EndLoopCalculator : public CalculatorBase {
using ItemT = typename IterableT::value_type;
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
RET_CHECK(cc->Inputs().HasTag("BATCH_END"))
<< "Missing BATCH_END tagged input_stream.";
cc->Inputs().Tag("BATCH_END").Set<Timestamp>();
RET_CHECK(cc->Inputs().HasTag("ITEM"));
cc->Inputs().Tag("ITEM").Set<ItemT>();
RET_CHECK(cc->Outputs().HasTag("ITERABLE"));
cc->Outputs().Tag("ITERABLE").Set<IterableT>();
return ::mediapipe::OkStatus();
}
::mediapipe::Status Process(CalculatorContext* cc) override {
if (!cc->Inputs().Tag("ITEM").IsEmpty()) {
if (!input_stream_collection_) {
input_stream_collection_.reset(new IterableT);
}
input_stream_collection_->push_back(
cc->Inputs().Tag("ITEM").template Get<ItemT>());
}
if (!cc->Inputs().Tag("BATCH_END").Value().IsEmpty()) { // flush signal
Timestamp loop_control_ts =
cc->Inputs().Tag("BATCH_END").template Get<Timestamp>();
if (input_stream_collection_) {
cc->Outputs()
.Tag("ITERABLE")
.Add(input_stream_collection_.release(), loop_control_ts);
} else {
// Since there is no collection, inform downstream calculators to not
// expect any packet by updating the timestamp bounds.
cc->Outputs()
.Tag("ITERABLE")
.SetNextTimestampBound(Timestamp(loop_control_ts.Value() + 1));
}
}
return ::mediapipe::OkStatus();
}
private:
std::unique_ptr<IterableT> input_stream_collection_;
};
} // namespace mediapipe
#endif // MEDIAPIPE_CALCULATORS_CORE_END_LOOP_CALCULATOR_H_
mediapipe/calculators/core/packet_resampler_calculator.cc
+26 -5
View File
@@ -74,6 +74,12 @@ class PacketResamplerCalculator : public CalculatorBase {
::mediapipe::Status Process(CalculatorContext* cc) override;
private:
// Calculates the first sampled timestamp that incorporates a jittering
// offset.
void InitializeNextOutputTimestampWithJitter();
// Calculates the next sampled timestamp that incorporates a jittering offset.
void UpdateNextOutputTimestampWithJitter();
// Logic for Process() when jitter_ != 0.0.
::mediapipe::Status ProcessWithJitter(CalculatorContext* cc);
@@ -233,6 +239,7 @@ TimestampDiff TimestampDiffFromSeconds(double seconds) {
<< Timestamp::kTimestampUnitsPerSecond;
frame_time_usec_ = static_cast<int64>(1000000.0 / frame_rate_);
video_header_.frame_rate = frame_rate_;
if (resampler_options.output_header() !=
@@ -295,6 +302,17 @@ TimestampDiff TimestampDiffFromSeconds(double seconds) {
return ::mediapipe::OkStatus();
}
void PacketResamplerCalculator::InitializeNextOutputTimestampWithJitter() {
next_output_timestamp_ =
first_timestamp_ + frame_time_usec_ * random_->RandFloat();
}
void PacketResamplerCalculator::UpdateNextOutputTimestampWithJitter() {
next_output_timestamp_ +=
frame_time_usec_ *
((1.0 - jitter_) + 2.0 * jitter_ * random_->RandFloat());
}
::mediapipe::Status PacketResamplerCalculator::ProcessWithJitter(
CalculatorContext* cc) {
RET_CHECK_GT(cc->InputTimestamp(), Timestamp::PreStream());
@@ -302,8 +320,13 @@ TimestampDiff TimestampDiffFromSeconds(double seconds) {
if (first_timestamp_ == Timestamp::Unset()) {
first_timestamp_ = cc->InputTimestamp();
next_output_timestamp_ =
first_timestamp_ + frame_time_usec_ * random_->RandFloat();
InitializeNextOutputTimestampWithJitter();
if (first_timestamp_ == next_output_timestamp_) {
OutputWithinLimits(
cc,
cc->Inputs().Get(input_data_id_).Value().At(next_output_timestamp_));
UpdateNextOutputTimestampWithJitter();
}
return ::mediapipe::OkStatus();
}
@@ -322,9 +345,7 @@ TimestampDiff TimestampDiffFromSeconds(double seconds) {
? last_packet_
: cc->Inputs().Get(input_data_id_).Value())
.At(next_output_timestamp_));
next_output_timestamp_ +=
frame_time_usec_ *
((1.0 - jitter_) + 2.0 * jitter_ * random_->RandFloat());
UpdateNextOutputTimestampWithJitter();
return ::mediapipe::OkStatus();
}
mediapipe/calculators/image/BUILD
+2 -2
View File
@@ -12,14 +12,14 @@
# See the License for the specific language governing permissions and
# limitations under the License.
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
licenses(["notice"]) # Apache 2.0
package(default_visibility = ["//visibility:private"])
exports_files(["LICENSE"])
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
proto_library(
name = "opencv_image_encoder_calculator_proto",
srcs = ["opencv_image_encoder_calculator.proto"],
mediapipe/calculators/tensorflow/BUILD
+49 -61
View File
@@ -13,12 +13,12 @@
# limitations under the License.
#
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
licenses(["notice"]) # Apache 2.0
package(default_visibility = ["//visibility:private"])
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
proto_library(
name = "graph_tensors_packet_generator_proto",
srcs = ["graph_tensors_packet_generator.proto"],
@@ -138,7 +138,7 @@ mediapipe_cc_proto_library(
srcs = ["image_frame_to_tensor_calculator.proto"],
cc_deps = [
"//mediapipe/framework:calculator_cc_proto",
"@org_tensorflow//tensorflow/core:protos_all_cc",
"@org_tensorflow//tensorflow/core:protos_all",
],
visibility = ["//visibility:public"],
deps = [":image_frame_to_tensor_calculator_proto"],
@@ -173,7 +173,7 @@ mediapipe_cc_proto_library(
srcs = ["pack_media_sequence_calculator.proto"],
cc_deps = [
"//mediapipe/framework:calculator_cc_proto",
"@org_tensorflow//tensorflow/core:protos_all_cc",
"@org_tensorflow//tensorflow/core:protos_all",
],
visibility = ["//visibility:public"],
deps = [":pack_media_sequence_calculator_proto"],
@@ -192,7 +192,7 @@ mediapipe_cc_proto_library(
srcs = ["tensorflow_session_from_frozen_graph_generator.proto"],
cc_deps = [
"//mediapipe/framework:packet_generator_cc_proto",
"@org_tensorflow//tensorflow/core:protos_all_cc",
"@org_tensorflow//tensorflow/core:protos_all",
],
visibility = ["//visibility:public"],
deps = [":tensorflow_session_from_frozen_graph_generator_proto"],
@@ -203,7 +203,7 @@ mediapipe_cc_proto_library(
srcs = ["tensorflow_session_from_frozen_graph_calculator.proto"],
cc_deps = [
"//mediapipe/framework:calculator_cc_proto",
"@org_tensorflow//tensorflow/core:protos_all_cc",
"@org_tensorflow//tensorflow/core:protos_all",
],
visibility = ["//visibility:public"],
deps = [":tensorflow_session_from_frozen_graph_calculator_proto"],
@@ -277,7 +277,7 @@ mediapipe_cc_proto_library(
srcs = ["vector_int_to_tensor_calculator_options.proto"],
cc_deps = [
"//mediapipe/framework:calculator_cc_proto",
"@org_tensorflow//tensorflow/core:protos_all_cc",
"@org_tensorflow//tensorflow/core:protos_all",
],
visibility = ["//visibility:public"],
deps = [":vector_int_to_tensor_calculator_options_proto"],
@@ -296,7 +296,7 @@ cc_library(
srcs = ["graph_tensors_packet_generator.cc"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/calculators/tensorflow:graph_tensors_packet_generator_cc_proto",
":graph_tensors_packet_generator_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
@@ -311,7 +311,7 @@ cc_library(
srcs = ["image_frame_to_tensor_calculator.cc"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/calculators/tensorflow:image_frame_to_tensor_calculator_cc_proto",
":image_frame_to_tensor_calculator_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/port:ret_check",
@@ -333,7 +333,7 @@ cc_library(
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework/formats:time_series_header_cc_proto",
"//mediapipe/calculators/tensorflow:matrix_to_tensor_calculator_options_cc_proto",
":matrix_to_tensor_calculator_options_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/formats:matrix",
"//mediapipe/framework/port:status",
@@ -354,7 +354,7 @@ cc_library(
srcs = ["lapped_tensor_buffer_calculator.cc"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/calculators/tensorflow:lapped_tensor_buffer_calculator_cc_proto",
":lapped_tensor_buffer_calculator_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
@@ -408,7 +408,7 @@ cc_library(
"//mediapipe/util/sequence:media_sequence",
"//mediapipe/util/sequence:media_sequence_util",
"@com_google_absl//absl/strings",
"@org_tensorflow//tensorflow/core:protos_all_cc",
"@org_tensorflow//tensorflow/core:protos_all",
],
alwayslink = 1,
)
@@ -423,7 +423,7 @@ cc_library(
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"@org_tensorflow//tensorflow/core:protos_all_cc",
"@org_tensorflow//tensorflow/core:protos_all",
],
alwayslink = 1,
)
@@ -436,7 +436,7 @@ cc_library(
visibility = ["//visibility:public"],
deps = [
":tensorflow_session",
"//mediapipe/calculators/tensorflow:tensorflow_inference_calculator_cc_proto",
":tensorflow_inference_calculator_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/tool:status_util",
"@com_google_absl//absl/strings",
@@ -514,7 +514,7 @@ cc_library(
visibility = ["//visibility:public"],
deps = [
":tensorflow_session",
"//mediapipe/calculators/tensorflow:tensorflow_session_from_frozen_graph_generator_cc_proto",
":tensorflow_session_from_frozen_graph_generator_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/tool:status_util",
"//mediapipe/framework/port:status",
@@ -573,7 +573,7 @@ cc_library(
visibility = ["//visibility:public"],
deps = [
":tensorflow_session",
"//mediapipe/calculators/tensorflow:tensorflow_session_from_saved_model_generator_cc_proto",
":tensorflow_session_from_saved_model_generator_cc_proto",
"//mediapipe/framework:packet_generator",
"//mediapipe/framework:packet_type",
"//mediapipe/framework/tool:status_util",
@@ -597,7 +597,7 @@ cc_library(
srcs = ["tensor_squeeze_dimensions_calculator.cc"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/calculators/tensorflow:tensor_squeeze_dimensions_calculator_cc_proto",
":tensor_squeeze_dimensions_calculator_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
@@ -611,7 +611,7 @@ cc_library(
srcs = ["tensor_to_image_frame_calculator.cc"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/calculators/tensorflow:tensor_to_image_frame_calculator_cc_proto",
":tensor_to_image_frame_calculator_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/port:ret_check",
@@ -627,7 +627,7 @@ cc_library(
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework/formats:time_series_header_cc_proto",
"//mediapipe/calculators/tensorflow:tensor_to_matrix_calculator_cc_proto",
":tensor_to_matrix_calculator_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/formats:matrix",
"//mediapipe/framework/port:status",
@@ -654,7 +654,7 @@ cc_library(
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"@org_tensorflow//tensorflow/core:lib",
"@org_tensorflow//tensorflow/core:protos_all_cc",
"@org_tensorflow//tensorflow/core:protos_all",
],
alwayslink = 1,
)
@@ -667,7 +667,7 @@ cc_library(
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/port:status",
"//mediapipe/framework/port:ret_check",
"//mediapipe/calculators/tensorflow:tensor_to_vector_float_calculator_options_cc_proto",
":tensor_to_vector_float_calculator_options_cc_proto",
] + select({
"//conditions:default": [
"@org_tensorflow//tensorflow/core:framework",
@@ -695,7 +695,7 @@ cc_library(
"//mediapipe/util:audio_decoder_cc_proto",
"//mediapipe/util/sequence:media_sequence",
"@com_google_absl//absl/strings",
"@org_tensorflow//tensorflow/core:protos_all_cc",
"@org_tensorflow//tensorflow/core:protos_all",
],
alwayslink = 1,
)
@@ -719,7 +719,7 @@ cc_library(
srcs = ["vector_float_to_tensor_calculator.cc"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/calculators/tensorflow:vector_float_to_tensor_calculator_options_cc_proto",
":vector_float_to_tensor_calculator_options_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
@@ -733,11 +733,11 @@ cc_library(
srcs = ["unpack_yt8m_sequence_example_calculator.cc"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/calculators/tensorflow:lapped_tensor_buffer_calculator_cc_proto",
":lapped_tensor_buffer_calculator_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:packet",
"//mediapipe/framework/port:status",
"@org_tensorflow//tensorflow/core:protos_all_cc",
"@org_tensorflow//tensorflow/core:protos_all",
],
alwayslink = 1,
)
@@ -747,7 +747,7 @@ cc_test(
srcs = ["graph_tensors_packet_generator_test.cc"],
deps = [
":graph_tensors_packet_generator",
"//mediapipe/calculators/tensorflow:graph_tensors_packet_generator_cc_proto",
":graph_tensors_packet_generator_cc_proto",
"//mediapipe/framework:packet",
"//mediapipe/framework:packet_generator_cc_proto",
"//mediapipe/framework:packet_set",
@@ -779,7 +779,7 @@ cc_test(
srcs = ["matrix_to_tensor_calculator_test.cc"],
deps = [
":matrix_to_tensor_calculator",
"//mediapipe/calculators/tensorflow:matrix_to_tensor_calculator_options_cc_proto",
":matrix_to_tensor_calculator_options_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:calculator_runner",
"//mediapipe/framework/formats:matrix",
@@ -795,13 +795,13 @@ cc_test(
srcs = ["lapped_tensor_buffer_calculator_test.cc"],
deps = [
":lapped_tensor_buffer_calculator",
"//mediapipe/calculators/tensorflow:lapped_tensor_buffer_calculator_cc_proto",
":lapped_tensor_buffer_calculator_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:calculator_runner",
"//mediapipe/framework/port:gtest_main",
"@com_google_absl//absl/memory",
"@org_tensorflow//tensorflow/core:framework",
"@org_tensorflow//tensorflow/core:protos_all_cc",
"@org_tensorflow//tensorflow/core:protos_all",
],
)
@@ -840,7 +840,7 @@ cc_test(
"//mediapipe/util/sequence:media_sequence",
"@com_google_absl//absl/memory",
"@com_google_absl//absl/strings",
"@org_tensorflow//tensorflow/core:protos_all_cc",
"@org_tensorflow//tensorflow/core:protos_all",
],
)
@@ -867,7 +867,7 @@ cc_test(
"@com_google_absl//absl/strings",
"@org_tensorflow//tensorflow/core:direct_session",
"@org_tensorflow//tensorflow/core:framework",
"@org_tensorflow//tensorflow/core:protos_all_cc",
"@org_tensorflow//tensorflow/core:protos_all",
"@org_tensorflow//tensorflow/core:testlib",
"@org_tensorflow//tensorflow/core/kernels:conv_ops",
"@org_tensorflow//tensorflow/core/kernels:math",
@@ -883,7 +883,7 @@ cc_test(
":tensorflow_inference_calculator",
":tensorflow_session",
":tensorflow_session_from_frozen_graph_generator",
"//mediapipe/calculators/tensorflow:tensorflow_session_from_frozen_graph_generator_cc_proto",
":tensorflow_session_from_frozen_graph_generator_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:packet",
"//mediapipe/framework:packet_generator_cc_proto",
@@ -897,7 +897,7 @@ cc_test(
"@com_google_absl//absl/strings",
"@org_tensorflow//tensorflow/core:direct_session",
"@org_tensorflow//tensorflow/core:framework",
"@org_tensorflow//tensorflow/core:protos_all_cc",
"@org_tensorflow//tensorflow/core:protos_all",
"@org_tensorflow//tensorflow/core:testlib",
"@org_tensorflow//tensorflow/core/kernels:conv_ops",
"@org_tensorflow//tensorflow/core/kernels:math",
@@ -913,7 +913,7 @@ cc_test(
":tensorflow_inference_calculator",
":tensorflow_session",
":tensorflow_session_from_saved_model_generator",
"//mediapipe/calculators/tensorflow:tensorflow_session_from_saved_model_generator_cc_proto",
":tensorflow_session_from_saved_model_generator_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:packet",
"//mediapipe/framework:packet_generator_cc_proto",
@@ -923,14 +923,8 @@ cc_test(
"//mediapipe/framework/tool:tag_map_helper",
"//mediapipe/framework/tool:validate_type",
"@com_google_absl//absl/strings",
"@org_tensorflow//tensorflow/core:all_kernels",
"@org_tensorflow//tensorflow/core:direct_session",
"@org_tensorflow//tensorflow/core/kernels:array",
"@org_tensorflow//tensorflow/core/kernels:bitcast_op",
"@org_tensorflow//tensorflow/core/kernels:conv_ops",
"@org_tensorflow//tensorflow/core/kernels:io",
"@org_tensorflow//tensorflow/core/kernels:state",
"@org_tensorflow//tensorflow/core/kernels:string",
"@org_tensorflow//tensorflow/core/kernels/data:tensor_dataset_op",
],
)
@@ -954,14 +948,8 @@ cc_test(
"//mediapipe/framework/tool:tag_map_helper",
"//mediapipe/framework/tool:validate_type",
"@com_google_absl//absl/strings",
"@org_tensorflow//tensorflow/core:all_kernels",
"@org_tensorflow//tensorflow/core:direct_session",
"@org_tensorflow//tensorflow/core/kernels:array",
"@org_tensorflow//tensorflow/core/kernels:bitcast_op",
"@org_tensorflow//tensorflow/core/kernels:conv_ops",
"@org_tensorflow//tensorflow/core/kernels:io",
"@org_tensorflow//tensorflow/core/kernels:state",
"@org_tensorflow//tensorflow/core/kernels:string",
"@org_tensorflow//tensorflow/core/kernels/data:tensor_dataset_op",
],
)
@@ -970,12 +958,12 @@ cc_test(
srcs = ["tensor_squeeze_dimensions_calculator_test.cc"],
deps = [
":tensor_squeeze_dimensions_calculator",
"//mediapipe/calculators/tensorflow:tensor_squeeze_dimensions_calculator_cc_proto",
":tensor_squeeze_dimensions_calculator_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:calculator_runner",
"//mediapipe/framework/port:gtest_main",
"@org_tensorflow//tensorflow/core:framework",
"@org_tensorflow//tensorflow/core:protos_all_cc",
"@org_tensorflow//tensorflow/core:protos_all",
],
)
@@ -985,13 +973,13 @@ cc_test(
srcs = ["tensor_to_image_frame_calculator_test.cc"],
deps = [
":tensor_to_image_frame_calculator",
"//mediapipe/calculators/tensorflow:tensor_to_image_frame_calculator_cc_proto",
":tensor_to_image_frame_calculator_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:calculator_runner",
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/port:gtest_main",
"@org_tensorflow//tensorflow/core:framework",
"@org_tensorflow//tensorflow/core:protos_all_cc",
"@org_tensorflow//tensorflow/core:protos_all",
],
)
@@ -1001,14 +989,14 @@ cc_test(
srcs = ["tensor_to_matrix_calculator_test.cc"],
deps = [
":tensor_to_matrix_calculator",
"//mediapipe/calculators/tensorflow:tensor_to_matrix_calculator_cc_proto",
":tensor_to_matrix_calculator_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:calculator_runner",
"//mediapipe/framework/formats:matrix",
"//mediapipe/framework/formats:time_series_header_cc_proto",
"//mediapipe/framework/port:gtest_main",
"@org_tensorflow//tensorflow/core:framework",
"@org_tensorflow//tensorflow/core:protos_all_cc",
"@org_tensorflow//tensorflow/core:protos_all",
],
)
@@ -1017,12 +1005,12 @@ cc_test(
srcs = ["tensor_to_vector_float_calculator_test.cc"],
deps = [
":tensor_to_vector_float_calculator",
"//mediapipe/calculators/tensorflow:tensor_to_vector_float_calculator_options_cc_proto",
":tensor_to_vector_float_calculator_options_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:calculator_runner",
"//mediapipe/framework/port:gtest_main",
"@org_tensorflow//tensorflow/core:framework",
"@org_tensorflow//tensorflow/core:protos_all_cc",
"@org_tensorflow//tensorflow/core:protos_all",
],
)
@@ -1042,7 +1030,7 @@ cc_test(
"//mediapipe/util/sequence:media_sequence",
"@com_google_absl//absl/memory",
"@com_google_absl//absl/strings",
"@org_tensorflow//tensorflow/core:protos_all_cc",
"@org_tensorflow//tensorflow/core:protos_all",
],
)
@@ -1056,7 +1044,7 @@ cc_test(
"//mediapipe/framework:calculator_runner",
"//mediapipe/framework/port:gtest_main",
"@org_tensorflow//tensorflow/core:framework",
"@org_tensorflow//tensorflow/core:protos_all_cc",
"@org_tensorflow//tensorflow/core:protos_all",
],
)
@@ -1065,12 +1053,12 @@ cc_test(
srcs = ["vector_float_to_tensor_calculator_test.cc"],
deps = [
":vector_float_to_tensor_calculator",
"//mediapipe/calculators/tensorflow:vector_float_to_tensor_calculator_options_cc_proto",
":vector_float_to_tensor_calculator_options_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:calculator_runner",
"//mediapipe/framework/port:gtest_main",
"@org_tensorflow//tensorflow/core:framework",
"@org_tensorflow//tensorflow/core:protos_all_cc",
"@org_tensorflow//tensorflow/core:protos_all",
],
)
@@ -1094,7 +1082,7 @@ cc_test(
":tensorflow_session",
":tensorflow_inference_calculator",
":tensorflow_session_from_frozen_graph_generator",
"//mediapipe/calculators/tensorflow:tensorflow_session_from_frozen_graph_generator_cc_proto",
":tensorflow_session_from_frozen_graph_generator_cc_proto",
"//mediapipe/framework/deps:file_path",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:calculator_runner",
mediapipe/calculators/tflite/BUILD
+1
View File
@@ -238,6 +238,7 @@ cc_library(
"@org_tensorflow//tensorflow/lite/delegates/gpu/common:shape",
"@org_tensorflow//tensorflow/lite/delegates/gpu/metal:buffer_convert",
"@org_tensorflow//tensorflow/lite/delegates/gpu:metal_delegate",
"@org_tensorflow//tensorflow/lite/delegates/gpu:metal_delegate_internal",
],
"//conditions:default": [
"//mediapipe/gpu:gl_calculator_helper",
mediapipe/calculators/tflite/tflite_inference_calculator.cc
+4 -4
View File
@@ -27,8 +27,7 @@
#include "tensorflow/lite/kernels/register.h"
#include "tensorflow/lite/model.h"
#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
!defined(__APPLE__)
#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
#include "mediapipe/gpu/gl_calculator_helper.h"
#include "mediapipe/gpu/gpu_buffer.h"
#include "tensorflow/lite/delegates/gpu/common/shape.h"
@@ -49,6 +48,7 @@
#include "tensorflow/lite/delegates/gpu/common/shape.h"
#include "tensorflow/lite/delegates/gpu/metal/buffer_convert.h"
#include "tensorflow/lite/delegates/gpu/metal_delegate.h"
#include "tensorflow/lite/delegates/gpu/metal_delegate_internal.h"
#endif // iOS
namespace {
@@ -553,9 +553,9 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
#if defined(__APPLE__) && !TARGET_OS_OSX // iOS
// Configure and create the delegate.
GpuDelegateOptions options;
TFLGpuDelegateOptions options;
options.allow_precision_loss = false; // Must match converter, F=float/T=half
options.wait_type = GpuDelegateOptions::WaitType::kPassive;
options.wait_type = TFLGpuDelegateWaitType::TFLGpuDelegateWaitTypePassive;
if (!delegate_) delegate_ = TFLGpuDelegateCreate(&options);
id<MTLDevice> device = gpu_helper_.mtlDevice;
mediapipe/calculators/tflite/tflite_tensors_to_detections_calculator.cc
+3 -13
View File
@@ -172,13 +172,11 @@ class TfLiteTensorsToDetectionsCalculator : public CalculatorBase {
const int* detection_classes, std::vector<Detection>* output_detections);
Detection ConvertToDetection(float box_ymin, float box_xmin, float box_ymax,
float box_xmax, float score, int class_id,
int detection_id, bool flip_vertically);
bool flip_vertically);
int num_classes_ = 0;
int num_boxes_ = 0;
int num_coords_ = 0;
// Unique detection ID per new detection.
static int next_detection_id_;
std::set<int> ignore_classes_;
::mediapipe::TfLiteTensorsToDetectionsCalculatorOptions options_;
@@ -199,10 +197,6 @@ class TfLiteTensorsToDetectionsCalculator : public CalculatorBase {
};
REGISTER_CALCULATOR(TfLiteTensorsToDetectionsCalculator);
// Initialization of non-const static member should happen outside class
// definition.
int TfLiteTensorsToDetectionsCalculator::next_detection_id_ = 0;
::mediapipe::Status TfLiteTensorsToDetectionsCalculator::GetContract(
CalculatorContract* cc) {
RET_CHECK(!cc->Inputs().GetTags().empty());
@@ -686,10 +680,7 @@ int TfLiteTensorsToDetectionsCalculator::next_detection_id_ = 0;
Detection detection = ConvertToDetection(
detection_boxes[box_offset + 0], detection_boxes[box_offset + 1],
detection_boxes[box_offset + 2], detection_boxes[box_offset + 3],
detection_scores[i], detection_classes[i], next_detection_id_,
options_.flip_vertically());
// Increment to get next unique detection ID.
++next_detection_id_;
detection_scores[i], detection_classes[i], options_.flip_vertically());
// Add keypoints.
if (options_.num_keypoints() > 0) {
auto* location_data = detection.mutable_location_data();
@@ -712,11 +703,10 @@ int TfLiteTensorsToDetectionsCalculator::next_detection_id_ = 0;
Detection TfLiteTensorsToDetectionsCalculator::ConvertToDetection(
float box_ymin, float box_xmin, float box_ymax, float box_xmax, float score,
int class_id, int detection_id, bool flip_vertically) {
int class_id, bool flip_vertically) {
Detection detection;
detection.add_score(score);
detection.add_label_id(class_id);
detection.set_detection_id(detection_id);
LocationData* location_data = detection.mutable_location_data();
location_data->set_format(LocationData::RELATIVE_BOUNDING_BOX);
mediapipe/calculators/util/BUILD
+138 -2
View File
@@ -12,14 +12,14 @@
# See the License for the specific language governing permissions and
# limitations under the License.
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
licenses(["notice"]) # Apache 2.0
package(default_visibility = ["//visibility:public"])
exports_files(["LICENSE"])
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
proto_library(
name = "annotation_overlay_calculator_proto",
srcs = ["annotation_overlay_calculator.proto"],
@@ -72,6 +72,24 @@ proto_library(
],
)
proto_library(
name = "collection_has_min_size_calculator_proto",
srcs = ["collection_has_min_size_calculator.proto"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_proto",
],
)
proto_library(
name = "association_calculator_proto",
srcs = ["association_calculator.proto"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_proto",
],
)
mediapipe_cc_proto_library(
name = "annotation_overlay_calculator_cc_proto",
srcs = ["annotation_overlay_calculator.proto"],
@@ -141,6 +159,26 @@ mediapipe_cc_proto_library(
],
)
mediapipe_cc_proto_library(
name = "collection_has_min_size_calculator_cc_proto",
srcs = ["collection_has_min_size_calculator.proto"],
cc_deps = [
"//mediapipe/framework:calculator_cc_proto",
],
visibility = ["//mediapipe:__subpackages__"],
deps = [":collection_has_min_size_calculator_proto"],
)
mediapipe_cc_proto_library(
name = "association_calculator_cc_proto",
srcs = ["association_calculator.proto"],
cc_deps = [
"//mediapipe/framework:calculator_cc_proto",
],
visibility = ["//mediapipe:__subpackages__"],
deps = [":association_calculator_proto"],
)
cc_library(
name = "packet_frequency_calculator",
srcs = ["packet_frequency_calculator.cc"],
@@ -847,3 +885,101 @@ cc_library(
],
alwayslink = 1,
)
cc_library(
name = "filter_collection_calculator",
srcs = ["filter_collection_calculator.cc"],
hdrs = ["filter_collection_calculator.h"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/formats:landmark_cc_proto",
"//mediapipe/framework/formats:rect_cc_proto",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"@com_google_absl//absl/strings",
],
alwayslink = 1,
)
cc_library(
name = "collection_has_min_size_calculator",
srcs = ["collection_has_min_size_calculator.cc"],
hdrs = ["collection_has_min_size_calculator.h"],
visibility = ["//visibility:public"],
deps = [
":collection_has_min_size_calculator_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/formats:rect_cc_proto",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
],
alwayslink = 1,
)
cc_library(
name = "association_calculator",
hdrs = ["association_calculator.h"],
visibility = ["//visibility:public"],
deps = [
":association_calculator_cc_proto",
"//mediapipe/framework:calculator_context",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:collection_item_id",
"//mediapipe/framework/port:rectangle",
"//mediapipe/framework/port:status",
"@com_google_absl//absl/memory",
],
alwayslink = 1,
)
cc_library(
name = "association_norm_rect_calculator",
srcs = ["association_norm_rect_calculator.cc"],
visibility = ["//visibility:public"],
deps = [
":association_calculator",
"//mediapipe/framework:calculator_context",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/formats:rect_cc_proto",
"//mediapipe/framework/port:rectangle",
"//mediapipe/framework/port:status",
],
alwayslink = 1,
)
cc_library(
name = "association_detection_calculator",
srcs = ["association_detection_calculator.cc"],
visibility = ["//visibility:public"],
deps = [
":association_calculator",
"//mediapipe/framework:calculator_context",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/formats:detection_cc_proto",
"//mediapipe/framework/formats:location",
"//mediapipe/framework/port:rectangle",
"//mediapipe/framework/port:status",
],
alwayslink = 1,
)
cc_test(
name = "association_calculator_test",
srcs = ["association_calculator_test.cc"],
deps = [
":association_detection_calculator",
":association_norm_rect_calculator",
"//mediapipe/framework:calculator_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:calculator_runner",
"//mediapipe/framework:collection_item_id",
"//mediapipe/framework:packet",
"//mediapipe/framework/deps:message_matchers",
"//mediapipe/framework/formats:detection_cc_proto",
"//mediapipe/framework/formats:location_data_cc_proto",
"//mediapipe/framework/formats:rect_cc_proto",
"//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:parse_text_proto",
],
)
mediapipe/calculators/util/annotation_overlay_calculator.cc
-10
View File
@@ -148,9 +148,6 @@ class AnnotationOverlayCalculator : public CalculatorBase {
// Underlying helper renderer library.
std::unique_ptr<AnnotationRenderer> renderer_;
// Number of input streams with render data.
int num_render_streams_;
// Indicates if image frame is available as input.
bool image_frame_available_ = false;
@@ -181,20 +178,15 @@ REGISTER_CALCULATOR(AnnotationOverlayCalculator);
return ::mediapipe::InternalError("GPU output must have GPU input.");
}
// Assume all inputs are render streams; adjust below.
int num_render_streams = cc->Inputs().NumEntries();
// Input image to render onto copy of.
#if !defined(MEDIAPIPE_DISABLE_GPU)
if (cc->Inputs().HasTag(kInputFrameTagGpu)) {
cc->Inputs().Tag(kInputFrameTagGpu).Set<mediapipe::GpuBuffer>();
num_render_streams = cc->Inputs().NumEntries() - 1;
use_gpu |= true;
}
#endif // !MEDIAPIPE_DISABLE_GPU
if (cc->Inputs().HasTag(kInputFrameTag)) {
cc->Inputs().Tag(kInputFrameTag).Set<ImageFrame>();
num_render_streams = cc->Inputs().NumEntries() - 1;
}
// Data streams to render.
@@ -246,12 +238,10 @@ REGISTER_CALCULATOR(AnnotationOverlayCalculator);
if (cc->Inputs().HasTag(kInputFrameTagGpu) ||
cc->Inputs().HasTag(kInputFrameTag)) {
image_frame_available_ = true;
num_render_streams_ = cc->Inputs().NumEntries() - 1;
} else {
image_frame_available_ = false;
RET_CHECK(options_.has_canvas_width_px());
RET_CHECK(options_.has_canvas_height_px());
num_render_streams_ = cc->Inputs().NumEntries();
}
// Initialize the helper renderer library.
mediapipe/calculators/util/association_calculator.h
+259
View File
@@ -0,0 +1,259 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef MEDIAPIPE_CALCULATORS_UTIL_ASSOCIATION_CALCULATOR_H_
#define MEDIAPIPE_CALCULATORS_UTIL_ASSOCIATION_CALCULATOR_H_
#include <memory>
#include <vector>
#include "absl/memory/memory.h"
#include "mediapipe/calculators/util/association_calculator.pb.h"
#include "mediapipe/framework/calculator_context.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/collection_item_id.h"
#include "mediapipe/framework/port/canonical_errors.h"
#include "mediapipe/framework/port/rectangle.h"
#include "mediapipe/framework/port/status.h"
namespace mediapipe {
// Computes the overlap similarity based on Intersection over Union (IoU) of
// two rectangles.
inline float OverlapSimilarity(const Rectangle_f& rect1,
const Rectangle_f& rect2) {
if (!rect1.Intersects(rect2)) return 0.0f;
// Compute IoU similarity score.
const float intersection_area = Rectangle_f(rect1).Intersect(rect2).Area();
const float normalization = rect1.Area() + rect2.Area() - intersection_area;
return normalization > 0.0f ? intersection_area / normalization : 0.0f;
}
// AssocationCalculator<T> accepts multiple inputs of vectors of type T that can
// be converted to Rectangle_f. The output is a vector of type T that contains
// elements from the input vectors that don't overlap with each other. When
// two elements overlap, the element that comes in from a later input stream
// is kept in the output. This association operation is useful for multiple
// instance inference pipelines in MediaPipe.
// If an input stream is tagged with "PREV" tag, IDs of overlapping elements
// from "PREV" input stream are propagated to the output. Elements in the "PREV"
// input stream that don't overlap with other elements are not added to the
// output. This stream is designed to take detections from previous timestamp,
// e.g. output of PreviousLoopbackCalculator to provide temporal association.
// See AssociationDetectionCalculator and AssociationNormRectCalculator for
// example uses.
template <typename T>
class AssociationCalculator : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
// Atmost one input stream can be tagged with "PREV".
RET_CHECK_LE(cc->Inputs().NumEntries("PREV"), 1);
if (cc->Inputs().HasTag("PREV")) {
RET_CHECK_GE(cc->Inputs().NumEntries(), 2);
}
for (CollectionItemId id = cc->Inputs().BeginId();
id < cc->Inputs().EndId(); ++id) {
cc->Inputs().Get(id).Set<std::vector<T>>();
}
cc->Outputs().Index(0).Set<std::vector<T>>();
return ::mediapipe::OkStatus();
}
::mediapipe::Status Open(CalculatorContext* cc) override {
cc->SetOffset(TimestampDiff(0));
has_prev_input_stream_ = cc->Inputs().HasTag("PREV");
if (has_prev_input_stream_) {
prev_input_stream_id_ = cc->Inputs().GetId("PREV", 0);
}
options_ = cc->Options<::mediapipe::AssociationCalculatorOptions>();
CHECK_GE(options_.min_similarity_threshold(), 0);
return ::mediapipe::OkStatus();
}
::mediapipe::Status Process(CalculatorContext* cc) override {
auto get_non_overlapping_elements = GetNonOverlappingElements(cc);
if (!get_non_overlapping_elements.ok()) {
return get_non_overlapping_elements.status();
}
std::list<T> result = get_non_overlapping_elements.ValueOrDie();
if (has_prev_input_stream_ &&
!cc->Inputs().Get(prev_input_stream_id_).IsEmpty()) {
// Processed all regular input streams. Now compare the result list
// elements with those in the PREV input stream, and propagate IDs from
// PREV input stream as appropriate.
const std::vector<T>& prev_input_vec =
cc->Inputs()
.Get(prev_input_stream_id_)
.template Get<std::vector<T>>();
MP_RETURN_IF_ERROR(
PropagateIdsFromPreviousToCurrent(prev_input_vec, &result));
}
auto output = absl::make_unique<std::vector<T>>();
for (auto it = result.begin(); it != result.end(); ++it) {
output->push_back(*it);
}
cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp());
return ::mediapipe::OkStatus();
}
protected:
::mediapipe::AssociationCalculatorOptions options_;
bool has_prev_input_stream_;
CollectionItemId prev_input_stream_id_;
virtual ::mediapipe::StatusOr<Rectangle_f> GetRectangle(const T& input) {
return ::mediapipe::OkStatus();
}
virtual std::pair<bool, int> GetId(const T& input) { return {false, -1}; }
virtual void SetId(T* input, int id) {}
private:
// Get a list of non-overlapping elements from all input streams, with
// increasing order of priority based on input stream index.
mediapipe::StatusOr<std::list<T>> GetNonOverlappingElements(
CalculatorContext* cc) {
std::list<T> result;
// Initialize result with the first non-empty input vector.
CollectionItemId non_empty_id = cc->Inputs().BeginId();
for (CollectionItemId id = cc->Inputs().BeginId();
id < cc->Inputs().EndId(); ++id) {
if (id == prev_input_stream_id_ || cc->Inputs().Get(id).IsEmpty()) {
continue;
}
const std::vector<T>& input_vec =
cc->Inputs().Get(id).Get<std::vector<T>>();
if (!input_vec.empty()) {
non_empty_id = id;
result.push_back(input_vec[0]);
for (int j = 1; j < input_vec.size(); ++j) {
MP_RETURN_IF_ERROR(AddElementToList(input_vec[j], &result));
}
break;
}
}
// Compare remaining input vectors with the non-empty result vector,
// remove lower-priority overlapping elements from the result vector and
// had corresponding higher-priority elements as necessary.
for (CollectionItemId id = non_empty_id + 1; id < cc->Inputs().EndId();
++id) {
if (id == prev_input_stream_id_ || cc->Inputs().Get(id).IsEmpty()) {
continue;
}
const std::vector<T>& input_vec =
cc->Inputs().Get(id).Get<std::vector<T>>();
for (int vi = 0; vi < input_vec.size(); ++vi) {
MP_RETURN_IF_ERROR(AddElementToList(input_vec[vi], &result));
}
}
return result;
}
::mediapipe::Status AddElementToList(T element, std::list<T>* current) {
// Compare this element with elements of the input collection. If this
// element has high overlap with elements of the collection, remove
// those elements from the collection and add this element.
ASSIGN_OR_RETURN(auto cur_rect, GetRectangle(element));
bool change_id = false;
int new_elem_id = -1;
for (auto uit = current->begin(); uit != current->end();) {
ASSIGN_OR_RETURN(auto prev_rect, GetRectangle(*uit));
if (OverlapSimilarity(cur_rect, prev_rect) >
options_.min_similarity_threshold()) {
std::pair<bool, int> prev_id = GetId(*uit);
// If prev_id.first is false when some element doesn't have an ID,
// change_id and new_elem_id will not be updated.
if (prev_id.first) {
change_id = prev_id.first;
new_elem_id = prev_id.second;
}
uit = current->erase(uit);
} else {
++uit;
}
}
if (change_id) {
SetId(&element, new_elem_id);
}
current->push_back(element);
return ::mediapipe::OkStatus();
}
// Compare elements of the current list with elements in from the collection
// of elements from the previous input stream, and propagate IDs from the
// previous input stream as appropriate.
::mediapipe::Status PropagateIdsFromPreviousToCurrent(
const std::vector<T>& prev_input_vec, std::list<T>* current) {
for (auto vit = current->begin(); vit != current->end(); ++vit) {
auto get_cur_rectangle = GetRectangle(*vit);
if (!get_cur_rectangle.ok()) {
return get_cur_rectangle.status();
}
const Rectangle_f& cur_rect = get_cur_rectangle.ValueOrDie();
bool change_id = false;
int id_for_vi = -1;
for (int ui = 0; ui < prev_input_vec.size(); ++ui) {
auto get_prev_rectangle = GetRectangle(prev_input_vec[ui]);
if (!get_prev_rectangle.ok()) {
return get_prev_rectangle.status();
}
const Rectangle_f& prev_rect = get_prev_rectangle.ValueOrDie();
if (OverlapSimilarity(cur_rect, prev_rect) >
options_.min_similarity_threshold()) {
std::pair<bool, int> prev_id = GetId(prev_input_vec[ui]);
// If prev_id.first is false when some element doesn't have an ID,
// change_id and id_for_vi will not be updated.
if (prev_id.first) {
change_id = prev_id.first;
id_for_vi = prev_id.second;
}
}
}
if (change_id) {
T element = *vit;
SetId(&element, id_for_vi);
*vit = element;
}
}
return ::mediapipe::OkStatus();
}
};
} // namespace mediapipe
#endif // MEDIAPIPE_CALCULATORS_UTIL_ASSOCIATION_CALCULATOR_H_
mediapipe/calculators/util/association_calculator.proto
+27
View File
@@ -0,0 +1,27 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto2";
package mediapipe;
import "mediapipe/framework/calculator.proto";
message AssociationCalculatorOptions {
extend CalculatorOptions {
optional AssociationCalculatorOptions ext = 275124847;
}
optional float min_similarity_threshold = 1 [default = 1.0];
}
mediapipe/calculators/util/association_calculator_test.cc
+476
View File
@@ -0,0 +1,476 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/framework/calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_runner.h"
#include "mediapipe/framework/collection_item_id.h"
#include "mediapipe/framework/deps/message_matchers.h"
#include "mediapipe/framework/formats/detection.pb.h"
#include "mediapipe/framework/formats/location_data.pb.h"
#include "mediapipe/framework/formats/rect.pb.h"
#include "mediapipe/framework/packet.h"
#include "mediapipe/framework/port/gmock.h"
#include "mediapipe/framework/port/gtest.h"
#include "mediapipe/framework/port/parse_text_proto.h"
#include "mediapipe/framework/port/status_matchers.h"
namespace mediapipe {
namespace {
::mediapipe::Detection DetectionWithRelativeLocationData(double xmin,
double ymin,
double width,
double height) {
::mediapipe::Detection detection;
::mediapipe::LocationData* location_data = detection.mutable_location_data();
location_data->set_format(::mediapipe::LocationData::RELATIVE_BOUNDING_BOX);
location_data->mutable_relative_bounding_box()->set_xmin(xmin);
location_data->mutable_relative_bounding_box()->set_ymin(ymin);
location_data->mutable_relative_bounding_box()->set_width(width);
location_data->mutable_relative_bounding_box()->set_height(height);
return detection;
}
} // namespace
class AssociationDetectionCalculatorTest : public ::testing::Test {
protected:
AssociationDetectionCalculatorTest() {
// 0.4 ================
// | | | |
// 0.3 ===================== | DET2 | |
// | | | DET1 | | | DET4 |
// 0.2 | DET0 | =========== ================
// | | | | | |
// 0.1 =====|=============== |
// | DET3 | | |
// 0.0 ================ |
// | DET5 |
// -0.1 ===========
// 0.1 0.2 0.3 0.4 0.5 0.6 0.7 0.8 0.9 1.0 1.1 1.2
// Detection det_0.
det_0 = DetectionWithRelativeLocationData(/*xmin=*/0.1, /*ymin=*/0.1,
/*width=*/0.2, /*height=*/0.2);
det_0.set_detection_id(0);
// Detection det_1.
det_1 = DetectionWithRelativeLocationData(/*xmin=*/0.3, /*ymin=*/0.1,
/*width=*/0.2, /*height=*/0.2);
det_1.set_detection_id(1);
// Detection det_2.
det_2 = DetectionWithRelativeLocationData(/*xmin=*/0.9, /*ymin=*/0.2,
/*width=*/0.2, /*height=*/0.2);
det_2.set_detection_id(2);
// Detection det_3.
det_3 = DetectionWithRelativeLocationData(/*xmin=*/0.2, /*ymin=*/0.0,
/*width=*/0.3, /*height=*/0.3);
det_3.set_detection_id(3);
// Detection det_4.
det_4 = DetectionWithRelativeLocationData(/*xmin=*/1.0, /*ymin=*/0.2,
/*width=*/0.2, /*height=*/0.2);
det_4.set_detection_id(4);
// Detection det_5.
det_5 = DetectionWithRelativeLocationData(/*xmin=*/0.3, /*ymin=*/-0.1,
/*width=*/0.3, /*height=*/0.3);
det_5.set_detection_id(5);
}
::mediapipe::Detection det_0, det_1, det_2, det_3, det_4, det_5;
};
TEST_F(AssociationDetectionCalculatorTest, DetectionAssocTest) {
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
calculator: "AssociationDetectionCalculator"
input_stream: "input_vec_0"
input_stream: "input_vec_1"
input_stream: "input_vec_2"
output_stream: "output_vec"
options {
[mediapipe.AssociationCalculatorOptions.ext] {
min_similarity_threshold: 0.1
}
}
)"));
// Input Stream 0: det_0, det_1, det_2.
auto input_vec_0 = absl::make_unique<std::vector<::mediapipe::Detection>>();
input_vec_0->push_back(det_0);
input_vec_0->push_back(det_1);
input_vec_0->push_back(det_2);
runner.MutableInputs()->Index(0).packets.push_back(
Adopt(input_vec_0.release()).At(Timestamp(1)));
// Input Stream 1: det_3, det_4.
auto input_vec_1 = absl::make_unique<std::vector<::mediapipe::Detection>>();
input_vec_1->push_back(det_3);
input_vec_1->push_back(det_4);
runner.MutableInputs()->Index(1).packets.push_back(
Adopt(input_vec_1.release()).At(Timestamp(1)));
// Input Stream 2: det_5.
auto input_vec_2 = absl::make_unique<std::vector<::mediapipe::Detection>>();
input_vec_2->push_back(det_5);
runner.MutableInputs()->Index(2).packets.push_back(
Adopt(input_vec_2.release()).At(Timestamp(1)));
MP_ASSERT_OK(runner.Run()) << "Calculator execution failed.";
const std::vector<Packet>& output = runner.Outputs().Index(0).packets;
EXPECT_EQ(1, output.size());
const auto& assoc_rects =
output[0].Get<std::vector<::mediapipe::Detection>>();
// det_3 overlaps with det_0, det_1 and det_5 overlaps with det_3. Since det_5
// is in the highest priority, we remove other rects. det_4 overlaps with
// det_2, and det_4 is higher priority, so we keep it. The final output
// therefore contains 2 elements.
EXPECT_EQ(2, assoc_rects.size());
// Outputs are in order of inputs, so det_4 is before det_5 in output vector.
// det_4 overlaps with det_2, so new id for det_4 is 2.
EXPECT_TRUE(assoc_rects[0].has_detection_id());
EXPECT_EQ(2, assoc_rects[0].detection_id());
det_4.set_detection_id(2);
EXPECT_THAT(assoc_rects[0], EqualsProto(det_4));
// det_3 overlaps with det_0, so new id for det_3 is 0.
// det_3 overlaps with det_1, so new id for det_3 is 1.
// det_5 overlaps with det_3, so new id for det_5 is 1.
EXPECT_TRUE(assoc_rects[1].has_detection_id());
EXPECT_EQ(1, assoc_rects[1].detection_id());
det_5.set_detection_id(1);
EXPECT_THAT(assoc_rects[1], EqualsProto(det_5));
}
TEST_F(AssociationDetectionCalculatorTest, DetectionAssocTestWithPrev) {
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
calculator: "AssociationDetectionCalculator"
input_stream: "PREV:input_vec_0"
input_stream: "input_vec_1"
output_stream: "output_vec"
options {
[mediapipe.AssociationCalculatorOptions.ext] {
min_similarity_threshold: 0.1
}
}
)"));
// Input Stream 0: det_3, det_4.
auto input_vec_0 = absl::make_unique<std::vector<::mediapipe::Detection>>();
input_vec_0->push_back(det_3);
input_vec_0->push_back(det_4);
CollectionItemId prev_input_stream_id =
runner.MutableInputs()->GetId("PREV", 0);
runner.MutableInputs()
->Get(prev_input_stream_id)
.packets.push_back(Adopt(input_vec_0.release()).At(Timestamp(1)));
// Input Stream 1: det_5.
auto input_vec_1 = absl::make_unique<std::vector<::mediapipe::Detection>>();
input_vec_1->push_back(det_5);
CollectionItemId input_stream_id = runner.MutableInputs()->GetId("", 0);
runner.MutableInputs()
->Get(input_stream_id)
.packets.push_back(Adopt(input_vec_1.release()).At(Timestamp(1)));
MP_ASSERT_OK(runner.Run()) << "Calculator execution failed.";
const std::vector<Packet>& output = runner.Outputs().Index(0).packets;
EXPECT_EQ(1, output.size());
const auto& assoc_rects =
output[0].Get<std::vector<::mediapipe::Detection>>();
// det_5 overlaps with det_3 and doesn't overlap with det_4. Since det_4 is
// in the PREV input stream, it doesn't get copied to the output, so the final
// output contains 1 element.
EXPECT_EQ(1, assoc_rects.size());
// det_5 overlaps with det_3, det_3 is in PREV, so new id for det_5 is 3.
EXPECT_TRUE(assoc_rects[0].has_detection_id());
EXPECT_EQ(3, assoc_rects[0].detection_id());
det_5.set_detection_id(3);
EXPECT_THAT(assoc_rects[0], EqualsProto(det_5));
}
TEST_F(AssociationDetectionCalculatorTest, DetectionAssocTestReverse) {
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
calculator: "AssociationDetectionCalculator"
input_stream: "input_vec_0"
input_stream: "input_vec_1"
input_stream: "input_vec_2"
output_stream: "output_vec"
options {
[mediapipe.AssociationCalculatorOptions.ext] {
min_similarity_threshold: 0.1
}
}
)"));
// Input Stream 0: det_5.
auto input_vec_0 = absl::make_unique<std::vector<::mediapipe::Detection>>();
input_vec_0->push_back(det_5);
runner.MutableInputs()->Index(0).packets.push_back(
Adopt(input_vec_0.release()).At(Timestamp(1)));
// Input Stream 1: det_3, det_4.
auto input_vec_1 = absl::make_unique<std::vector<::mediapipe::Detection>>();
input_vec_1->push_back(det_3);
input_vec_1->push_back(det_4);
runner.MutableInputs()->Index(1).packets.push_back(
Adopt(input_vec_1.release()).At(Timestamp(1)));
// Input Stream 2: det_0, det_1, det_2.
auto input_vec_2 = absl::make_unique<std::vector<::mediapipe::Detection>>();
input_vec_2->push_back(det_0);
input_vec_2->push_back(det_1);
input_vec_2->push_back(det_2);
runner.MutableInputs()->Index(2).packets.push_back(
Adopt(input_vec_2.release()).At(Timestamp(1)));
MP_ASSERT_OK(runner.Run()) << "Calculator execution failed.";
const std::vector<Packet>& output = runner.Outputs().Index(0).packets;
EXPECT_EQ(1, output.size());
const auto& assoc_rects =
output[0].Get<std::vector<::mediapipe::Detection>>();
// det_3 overlaps with det_5, so det_5 is removed. det_0 overlaps with det_3,
// so det_3 is removed as det_0 is in higher priority for keeping. det_2
// overlaps with det_4 so det_4 is removed as det_2 is higher priority for
// keeping. The final output therefore contains 3 elements.
EXPECT_EQ(3, assoc_rects.size());
// Outputs are in same order as inputs.
// det_3 overlaps with det_5, so new id for det_3 is 5.
// det_0 overlaps with det_3, so new id for det_0 is 5.
EXPECT_TRUE(assoc_rects[0].has_detection_id());
EXPECT_EQ(5, assoc_rects[0].detection_id());
det_0.set_detection_id(5);
EXPECT_THAT(assoc_rects[0], EqualsProto(det_0));
// det_1 stays with id 1.
EXPECT_TRUE(assoc_rects[1].has_detection_id());
EXPECT_EQ(1, assoc_rects[1].detection_id());
EXPECT_THAT(assoc_rects[1], EqualsProto(det_1));
// det_2 overlaps with det_4, so new id for det_2 is 4.
EXPECT_TRUE(assoc_rects[2].has_detection_id());
EXPECT_EQ(4, assoc_rects[2].detection_id());
det_2.set_detection_id(4);
EXPECT_THAT(assoc_rects[2], EqualsProto(det_2));
}
class AssociationNormRectCalculatorTest : public ::testing::Test {
protected:
AssociationNormRectCalculatorTest() {
// 0.4 ================
// | | | |
// 0.3 ===================== | NR2 | |
// | | | NR1 | | | NR4 |
// 0.2 | NR0 | =========== ================
// | | | | | |
// 0.1 =====|=============== |
// | NR3 | | |
// 0.0 ================ |
// | NR5 |
// -0.1 ===========
// 0.1 0.2 0.3 0.4 0.5 0.6 0.7 0.8 0.9 1.0 1.1 1.2
// NormalizedRect nr_0.
nr_0.set_x_center(0.2);
nr_0.set_y_center(0.2);
nr_0.set_width(0.2);
nr_0.set_height(0.2);
// NormalizedRect nr_1.
nr_1.set_x_center(0.4);
nr_1.set_y_center(0.2);
nr_1.set_width(0.2);
nr_1.set_height(0.2);
// NormalizedRect nr_2.
nr_2.set_x_center(1.0);
nr_2.set_y_center(0.3);
nr_2.set_width(0.2);
nr_2.set_height(0.2);
// NormalizedRect nr_3.
nr_3.set_x_center(0.35);
nr_3.set_y_center(0.15);
nr_3.set_width(0.3);
nr_3.set_height(0.3);
// NormalizedRect nr_4.
nr_4.set_x_center(1.1);
nr_4.set_y_center(0.3);
nr_4.set_width(0.2);
nr_4.set_height(0.2);
// NormalizedRect nr_5.
nr_5.set_x_center(0.45);
nr_5.set_y_center(0.05);
nr_5.set_width(0.3);
nr_5.set_height(0.3);
}
::mediapipe::NormalizedRect nr_0, nr_1, nr_2, nr_3, nr_4, nr_5;
};
TEST_F(AssociationNormRectCalculatorTest, NormRectAssocTest) {
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
calculator: "AssociationNormRectCalculator"
input_stream: "input_vec_0"
input_stream: "input_vec_1"
input_stream: "input_vec_2"
output_stream: "output_vec"
options {
[mediapipe.AssociationCalculatorOptions.ext] {
min_similarity_threshold: 0.1
}
}
)"));
// Input Stream 0: nr_0, nr_1, nr_2.
auto input_vec_0 =
absl::make_unique<std::vector<::mediapipe::NormalizedRect>>();
input_vec_0->push_back(nr_0);
input_vec_0->push_back(nr_1);
input_vec_0->push_back(nr_2);
runner.MutableInputs()->Index(0).packets.push_back(
Adopt(input_vec_0.release()).At(Timestamp(1)));
// Input Stream 1: nr_3, nr_4.
auto input_vec_1 =
absl::make_unique<std::vector<::mediapipe::NormalizedRect>>();
input_vec_1->push_back(nr_3);
input_vec_1->push_back(nr_4);
runner.MutableInputs()->Index(1).packets.push_back(
Adopt(input_vec_1.release()).At(Timestamp(1)));
// Input Stream 2: nr_5.
auto input_vec_2 =
absl::make_unique<std::vector<::mediapipe::NormalizedRect>>();
input_vec_2->push_back(nr_5);
runner.MutableInputs()->Index(2).packets.push_back(
Adopt(input_vec_2.release()).At(Timestamp(1)));
MP_ASSERT_OK(runner.Run()) << "Calculator execution failed.";
const std::vector<Packet>& output = runner.Outputs().Index(0).packets;
EXPECT_EQ(1, output.size());
const auto& assoc_rects =
output[0].Get<std::vector<::mediapipe::NormalizedRect>>();
// nr_3 overlaps with nr_0, nr_1 and nr_5 overlaps with nr_3. Since nr_5 is
// in the highest priority, we remove other rects.
// nr_4 overlaps with nr_2, and nr_4 is higher priority, so we keep it.
// The final output therefore contains 2 elements.
EXPECT_EQ(2, assoc_rects.size());
// Outputs are in order of inputs, so nr_4 is before nr_5 in output vector.
EXPECT_THAT(assoc_rects[0], EqualsProto(nr_4));
EXPECT_THAT(assoc_rects[1], EqualsProto(nr_5));
}
TEST_F(AssociationNormRectCalculatorTest, NormRectAssocTestReverse) {
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
calculator: "AssociationNormRectCalculator"
input_stream: "input_vec_0"
input_stream: "input_vec_1"
input_stream: "input_vec_2"
output_stream: "output_vec"
options {
[mediapipe.AssociationCalculatorOptions.ext] {
min_similarity_threshold: 0.1
}
}
)"));
// Input Stream 0: nr_5.
auto input_vec_0 =
absl::make_unique<std::vector<::mediapipe::NormalizedRect>>();
input_vec_0->push_back(nr_5);
runner.MutableInputs()->Index(0).packets.push_back(
Adopt(input_vec_0.release()).At(Timestamp(1)));
// Input Stream 1: nr_3, nr_4.
auto input_vec_1 =
absl::make_unique<std::vector<::mediapipe::NormalizedRect>>();
input_vec_1->push_back(nr_3);
input_vec_1->push_back(nr_4);
runner.MutableInputs()->Index(1).packets.push_back(
Adopt(input_vec_1.release()).At(Timestamp(1)));
// Input Stream 2: nr_0, nr_1, nr_2.
auto input_vec_2 =
absl::make_unique<std::vector<::mediapipe::NormalizedRect>>();
input_vec_2->push_back(nr_0);
input_vec_2->push_back(nr_1);
input_vec_2->push_back(nr_2);
runner.MutableInputs()->Index(2).packets.push_back(
Adopt(input_vec_2.release()).At(Timestamp(1)));
MP_ASSERT_OK(runner.Run()) << "Calculator execution failed.";
const std::vector<Packet>& output = runner.Outputs().Index(0).packets;
EXPECT_EQ(1, output.size());
const auto& assoc_rects =
output[0].Get<std::vector<::mediapipe::NormalizedRect>>();
// nr_3 overlaps with nr_5, so nr_5 is removed. nr_0 overlaps with nr_3, so
// nr_3 is removed as nr_0 is in higher priority for keeping. nr_2 overlaps
// with nr_4 so nr_4 is removed as nr_2 is higher priority for keeping.
// The final output therefore contains 3 elements.
EXPECT_EQ(3, assoc_rects.size());
// Outputs are in same order as inputs.
EXPECT_THAT(assoc_rects[0], EqualsProto(nr_0));
EXPECT_THAT(assoc_rects[1], EqualsProto(nr_1));
EXPECT_THAT(assoc_rects[2], EqualsProto(nr_2));
}
TEST_F(AssociationNormRectCalculatorTest, NormRectAssocSingleInputStream) {
CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
calculator: "AssociationNormRectCalculator"
input_stream: "input_vec"
output_stream: "output_vec"
options {
[mediapipe.AssociationCalculatorOptions.ext] {
min_similarity_threshold: 0.1
}
}
)"));
// Input Stream : nr_3, nr_5.
auto input_vec =
absl::make_unique<std::vector<::mediapipe::NormalizedRect>>();
input_vec->push_back(nr_3);
input_vec->push_back(nr_5);
runner.MutableInputs()->Index(0).packets.push_back(
Adopt(input_vec.release()).At(Timestamp(1)));
MP_ASSERT_OK(runner.Run()) << "Calculator execution failed.";
const std::vector<Packet>& output = runner.Outputs().Index(0).packets;
EXPECT_EQ(1, output.size());
const auto& assoc_rects =
output[0].Get<std::vector<::mediapipe::NormalizedRect>>();
// nr_5 overlaps with nr_3. Since nr_5 is after nr_3 in the same input stream
// we remove nr_3 and keep nr_5.
// The final output therefore contains 1 elements.
EXPECT_EQ(1, assoc_rects.size());
EXPECT_THAT(assoc_rects[0], EqualsProto(nr_5));
}
} // namespace mediapipe
mediapipe/calculators/util/association_detection_calculator.cc
+77
View File
@@ -0,0 +1,77 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/calculators/util/association_calculator.h"
#include "mediapipe/framework/calculator_context.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/detection.pb.h"
#include "mediapipe/framework/formats/location.h"
#include "mediapipe/framework/port/rectangle.h"
#include "mediapipe/framework/port/status.h"
namespace mediapipe {
// A subclass of AssociationCalculator<T> for Detection. Example:
// node {
// calculator: "AssociationDetectionCalculator"
// input_stream: "PREV:input_vec_0"
// input_stream: "input_vec_1"
// input_stream: "input_vec_2"
// output_stream: "output_vec"
// options {
// [mediapipe.AssociationCalculatorOptions.ext] {
// min_similarity_threshold: 0.1
// }
// }
class AssociationDetectionCalculator
: public AssociationCalculator<::mediapipe::Detection> {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
return AssociationCalculator<::mediapipe::Detection>::GetContract(cc);
}
::mediapipe::Status Open(CalculatorContext* cc) override {
return AssociationCalculator<::mediapipe::Detection>::Open(cc);
}
::mediapipe::Status Process(CalculatorContext* cc) override {
return AssociationCalculator<::mediapipe::Detection>::Process(cc);
}
::mediapipe::Status Close(CalculatorContext* cc) override {
return AssociationCalculator<::mediapipe::Detection>::Close(cc);
}
protected:
::mediapipe::StatusOr<Rectangle_f> GetRectangle(
const ::mediapipe::Detection& input) override {
if (!input.has_location_data()) {
return ::mediapipe::InternalError("Missing location_data in Detection");
}
const Location location(input.location_data());
return location.GetRelativeBBox();
}
std::pair<bool, int> GetId(const ::mediapipe::Detection& input) override {
return {input.has_detection_id(), input.detection_id()};
}
void SetId(::mediapipe::Detection* input, int id) override {
input->set_detection_id(id);
}
};
REGISTER_CALCULATOR(AssociationDetectionCalculator);
} // namespace mediapipe
mediapipe/calculators/util/association_norm_rect_calculator.cc
+72
View File
@@ -0,0 +1,72 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/calculators/util/association_calculator.h"
#include "mediapipe/framework/calculator_context.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/rect.pb.h"
#include "mediapipe/framework/port/rectangle.h"
#include "mediapipe/framework/port/status.h"
namespace mediapipe {
// A subclass of AssociationCalculator<T> for NormalizedRect. Example use case:
// node {
// calculator: "AssociationNormRectCalculator"
// input_stream: "input_vec_0"
// input_stream: "input_vec_1"
// input_stream: "input_vec_2"
// output_stream: "output_vec"
// options {
// [mediapipe.AssociationCalculatorOptions.ext] {
// min_similarity_threshold: 0.1
// }
// }
class AssociationNormRectCalculator
: public AssociationCalculator<::mediapipe::NormalizedRect> {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
return AssociationCalculator<::mediapipe::NormalizedRect>::GetContract(cc);
}
::mediapipe::Status Open(CalculatorContext* cc) override {
return AssociationCalculator<::mediapipe::NormalizedRect>::Open(cc);
}
::mediapipe::Status Process(CalculatorContext* cc) override {
return AssociationCalculator<::mediapipe::NormalizedRect>::Process(cc);
}
::mediapipe::Status Close(CalculatorContext* cc) override {
return AssociationCalculator<::mediapipe::NormalizedRect>::Close(cc);
}
protected:
::mediapipe::StatusOr<Rectangle_f> GetRectangle(
const ::mediapipe::NormalizedRect& input) override {
if (!input.has_x_center() || !input.has_y_center() || !input.has_width() ||
!input.has_height()) {
return ::mediapipe::InternalError(
"Missing dimensions in NormalizedRect.");
}
const float xmin = input.x_center() - input.width() / 2.0;
const float ymin = input.y_center() - input.height() / 2.0;
// TODO: Support rotation for rectangle.
return Rectangle_f(xmin, ymin, input.width(), input.height());
}
};
REGISTER_CALCULATOR(AssociationNormRectCalculator);
} // namespace mediapipe
mediapipe/calculators/util/collection_has_min_size_calculator.cc
+26
View File
@@ -0,0 +1,26 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/calculators/util/collection_has_min_size_calculator.h"
#include "mediapipe/framework/formats/rect.pb.h"
namespace mediapipe {
typedef CollectionHasMinSizeCalculator<std::vector<::mediapipe::NormalizedRect>>
NormalizedRectVectorHasMinSizeCalculator;
REGISTER_CALCULATOR(NormalizedRectVectorHasMinSizeCalculator);
} // namespace mediapipe
mediapipe/calculators/util/collection_has_min_size_calculator.h
+84
View File
@@ -0,0 +1,84 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef MEDIAPIPE_CALCULATORS_UTIL_COLLECTION_HAS_MIN_SIZE_CALCULATOR_H_
#define MEDIAPIPE_CALCULATORS_UTIL_COLLECTION_HAS_MIN_SIZE_CALCULATOR_H_
#include <vector>
#include "mediapipe/calculators/util/collection_has_min_size_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/port/canonical_errors.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status.h"
namespace mediapipe {
// Deterimines if an input iterable collection has a minimum size, specified
// in CollectionHasMinSizeCalculatorOptions. Example usage:
// node {
// calculator: "IntVectorHasMinSizeCalculator"
// input_stream: "ITERABLE:input_int_vector"
// output_stream: "has_min_ints"
// options {
// [mediapipe.CollectionHasMinSizeCalculatorOptions.ext] {
// min_size: 2
// }
// }
// }
template <typename IterableT>
class CollectionHasMinSizeCalculator : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
RET_CHECK(cc->Inputs().HasTag("ITERABLE"));
RET_CHECK_EQ(1, cc->Inputs().NumEntries());
RET_CHECK_EQ(1, cc->Outputs().NumEntries());
RET_CHECK_GE(
cc->Options<::mediapipe::CollectionHasMinSizeCalculatorOptions>()
.min_size(),
0);
cc->Inputs().Tag("ITERABLE").Set<IterableT>();
cc->Outputs().Index(0).Set<bool>();
return ::mediapipe::OkStatus();
}
::mediapipe::Status Open(CalculatorContext* cc) override {
cc->SetOffset(TimestampDiff(0));
min_size_ =
cc->Options<::mediapipe::CollectionHasMinSizeCalculatorOptions>()
.min_size();
return ::mediapipe::OkStatus();
}
::mediapipe::Status Process(CalculatorContext* cc) override {
const IterableT& input = cc->Inputs().Tag("ITERABLE").Get<IterableT>();
bool has_min_size = input.size() >= min_size_;
cc->Outputs().Index(0).AddPacket(
MakePacket<bool>(has_min_size).At(cc->InputTimestamp()));
return ::mediapipe::OkStatus();
}
private:
int min_size_ = 0;
};
} // namespace mediapipe
#endif // MEDIAPIPE_CALCULATORS_UTIL_COLLECTION_HAS_MIN_SIZE_CALCULATOR_H_
mediapipe/calculators/util/collection_has_min_size_calculator.proto
+29
View File
@@ -0,0 +1,29 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto2";
package mediapipe;
import "mediapipe/framework/calculator.proto";
message CollectionHasMinSizeCalculatorOptions {
extend CalculatorOptions {
optional CollectionHasMinSizeCalculatorOptions ext = 259397840;
}
// The minimum size an input iterable collection should have for the
// calculator to output true.
optional int32 min_size = 1 [default = 0];
}
mediapipe/calculators/util/filter_collection_calculator.cc
+34
View File
@@ -0,0 +1,34 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/calculators/util/filter_collection_calculator.h"
#include <vector>
#include "mediapipe/framework/formats/landmark.pb.h"
#include "mediapipe/framework/formats/rect.pb.h"
namespace mediapipe {
typedef FilterCollectionCalculator<std::vector<::mediapipe::NormalizedRect>>
FilterNormalizedRectCollectionCalculator;
REGISTER_CALCULATOR(FilterNormalizedRectCollectionCalculator);
typedef FilterCollectionCalculator<
std::vector<std::vector<::mediapipe::NormalizedLandmark>>>
FilterLandmarksCollectionCalculator;
REGISTER_CALCULATOR(FilterLandmarksCollectionCalculator);
} // namespace mediapipe
mediapipe/calculators/util/filter_collection_calculator.h
+109
View File
@@ -0,0 +1,109 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef MEDIAPIPE_CALCULATORS_UTIL_FILTER_VECTOR_CALCULATOR_H_
#define MEDIAPIPE_CALCULATORS_UTIL_FILTER_VECTOR_CALCULATOR_H_
#include <vector>
#include "absl/strings/str_cat.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/port/canonical_errors.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status.h"
namespace mediapipe {
// A calculator that gates elements of an input collection based on
// corresponding boolean values of the "CONDITION" vector. If there is no input
// collection or "CONDITION" vector, the calculator forwards timestamp bounds
// for downstream calculators. If the "CONDITION" vector has false values for
// all elements of the input collection, the calculator outputs a packet
// containing an empty collection.
// Example usage:
// node {
// calculator: "FilterCollectionCalculator"
// input_stream: "ITERABLE:input_collection"
// input_stream: "CONDITION:condition_vector"
// output_stream: "ITERABLE:output_collection"
// }
// This calculator is able to handle collections of copyable types T.
template <typename IterableT>
class FilterCollectionCalculator : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
RET_CHECK(cc->Inputs().HasTag("ITERABLE"));
RET_CHECK(cc->Inputs().HasTag("CONDITION"));
RET_CHECK(cc->Outputs().HasTag("ITERABLE"));
cc->Inputs().Tag("ITERABLE").Set<IterableT>();
cc->Inputs().Tag("CONDITION").Set<std::vector<bool>>();
cc->Outputs().Tag("ITERABLE").Set<IterableT>();
return ::mediapipe::OkStatus();
}
::mediapipe::Status Open(CalculatorContext* cc) override {
cc->SetOffset(TimestampDiff(0));
return ::mediapipe::OkStatus();
}
::mediapipe::Status Process(CalculatorContext* cc) override {
if (cc->Inputs().Tag("ITERABLE").IsEmpty()) {
return ::mediapipe::OkStatus();
}
if (cc->Inputs().Tag("CONDITION").IsEmpty()) {
return ::mediapipe::OkStatus();
}
const std::vector<bool>& filter_by =
cc->Inputs().Tag("CONDITION").Get<std::vector<bool>>();
return FilterCollection<IterableT>(
std::is_copy_constructible<typename IterableT::value_type>(), cc,
filter_by);
}
template <typename IterableU>
::mediapipe::Status FilterCollection(std::true_type, CalculatorContext* cc,
const std::vector<bool>& filter_by) {
const IterableU& input = cc->Inputs().Tag("ITERABLE").Get<IterableU>();
if (input.size() != filter_by.size()) {
return ::mediapipe::InternalError(absl::StrCat(
"Input vector size: ", input.size(),
" doesn't mach condition vector size: ", filter_by.size()));
}
auto output = absl::make_unique<IterableU>();
for (int i = 0; i < input.size(); ++i) {
if (filter_by[i]) {
output->push_back(input[i]);
}
}
cc->Outputs().Tag("ITERABLE").Add(output.release(), cc->InputTimestamp());
return ::mediapipe::OkStatus();
}
template <typename IterableU>
::mediapipe::Status FilterCollection(std::false_type, CalculatorContext* cc,
const std::vector<bool>& filter_by) {
return ::mediapipe::InternalError(
"Cannot copy input collection to filter it.");
}
};
} // namespace mediapipe
#endif // MEDIAPIPE_CALCULATORS_UTIL_FILTER_VECTOR_CALCULATOR_H_
mediapipe/calculators/util/rect_transformation_calculator.cc
+43 -2
View File
@@ -23,7 +23,9 @@ namespace mediapipe {
namespace {
constexpr char kNormRectTag[] = "NORM_RECT";
constexpr char kNormRectsTag[] = "NORM_RECTS";
constexpr char kRectTag[] = "RECT";
constexpr char kRectsTag[] = "RECTS";
constexpr char kImageSizeTag[] = "IMAGE_SIZE";
// Wraps around an angle in radians to within -M_PI and M_PI.
@@ -72,17 +74,31 @@ REGISTER_CALCULATOR(RectTransformationCalculator);
::mediapipe::Status RectTransformationCalculator::GetContract(
CalculatorContract* cc) {
RET_CHECK(cc->Inputs().HasTag(kNormRectTag) ^ cc->Inputs().HasTag(kRectTag));
RET_CHECK_EQ((cc->Inputs().HasTag(kNormRectTag) ? 1 : 0) +
(cc->Inputs().HasTag(kNormRectsTag) ? 1 : 0) +
(cc->Inputs().HasTag(kRectTag) ? 1 : 0) +
(cc->Inputs().HasTag(kRectsTag) ? 1 : 0),
1);
if (cc->Inputs().HasTag(kRectTag)) {
cc->Inputs().Tag(kRectTag).Set<Rect>();
cc->Outputs().Index(0).Set<Rect>();
}
if (cc->Inputs().HasTag(kRectsTag)) {
cc->Inputs().Tag(kRectsTag).Set<std::vector<Rect>>();
cc->Outputs().Index(0).Set<std::vector<Rect>>();
}
if (cc->Inputs().HasTag(kNormRectTag)) {
RET_CHECK(cc->Inputs().HasTag(kImageSizeTag));
cc->Inputs().Tag(kNormRectTag).Set<NormalizedRect>();
cc->Inputs().Tag(kImageSizeTag).Set<std::pair<int, int>>();
cc->Outputs().Index(0).Set<NormalizedRect>();
}
if (cc->Inputs().HasTag(kNormRectsTag)) {
RET_CHECK(cc->Inputs().HasTag(kImageSizeTag));
cc->Inputs().Tag(kNormRectsTag).Set<std::vector<NormalizedRect>>();
cc->Inputs().Tag(kImageSizeTag).Set<std::pair<int, int>>();
cc->Outputs().Index(0).Set<std::vector<NormalizedRect>>();
}
return ::mediapipe::OkStatus();
}
@@ -105,7 +121,17 @@ REGISTER_CALCULATOR(RectTransformationCalculator);
cc->Outputs().Index(0).AddPacket(
MakePacket<Rect>(rect).At(cc->InputTimestamp()));
}
if (cc->Inputs().HasTag(kRectsTag) &&
!cc->Inputs().Tag(kRectsTag).IsEmpty()) {
auto rects = cc->Inputs().Tag(kRectsTag).Get<std::vector<Rect>>();
auto output_rects = absl::make_unique<std::vector<Rect>>(rects.size());
for (int i = 0; i < rects.size(); ++i) {
output_rects->at(i) = rects[i];
auto it = output_rects->begin() + i;
TransformRect(&(*it));
}
cc->Outputs().Index(0).Add(output_rects.release(), cc->InputTimestamp());
}
if (cc->Inputs().HasTag(kNormRectTag) &&
!cc->Inputs().Tag(kNormRectTag).IsEmpty()) {
auto rect = cc->Inputs().Tag(kNormRectTag).Get<NormalizedRect>();
@@ -115,6 +141,21 @@ REGISTER_CALCULATOR(RectTransformationCalculator);
cc->Outputs().Index(0).AddPacket(
MakePacket<NormalizedRect>(rect).At(cc->InputTimestamp()));
}
if (cc->Inputs().HasTag(kNormRectsTag) &&
!cc->Inputs().Tag(kNormRectsTag).IsEmpty()) {
auto rects =
cc->Inputs().Tag(kNormRectsTag).Get<std::vector<NormalizedRect>>();
const auto& image_size =
cc->Inputs().Tag(kImageSizeTag).Get<std::pair<int, int>>();
auto output_rects =
absl::make_unique<std::vector<NormalizedRect>>(rects.size());
for (int i = 0; i < rects.size(); ++i) {
output_rects->at(i) = rects[i];
auto it = output_rects->begin() + i;
TransformNormalizedRect(&(*it), image_size.first, image_size.second);
}
cc->Outputs().Index(0).Add(output_rects.release(), cc->InputTimestamp());
}
return ::mediapipe::OkStatus();
}
mediapipe/calculators/video/BUILD
+19 -8
View File
@@ -13,12 +13,16 @@
# limitations under the License.
#
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
load(
"//mediapipe/framework/tool:mediapipe_graph.bzl",
"mediapipe_binary_graph",
)
licenses(["notice"]) # Apache 2.0
package(default_visibility = ["//visibility:private"])
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
proto_library(
name = "flow_to_image_calculator_proto",
srcs = ["flow_to_image_calculator.proto"],
@@ -52,9 +56,7 @@ mediapipe_cc_proto_library(
cc_library(
name = "flow_to_image_calculator",
srcs = ["flow_to_image_calculator.cc"],
visibility = [
"//visibility:public",
],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/calculators/video:flow_to_image_calculator_cc_proto",
"//mediapipe/calculators/video/tool:flow_quantizer_model",
@@ -129,10 +131,20 @@ cc_library(
alwayslink = 1,
)
filegroup(
name = "test_videos",
srcs = [
"testdata/format_FLV_H264_AAC.video",
"testdata/format_MKV_VP8_VORBIS.video",
"testdata/format_MP4_AVC720P_AAC.video",
],
visibility = ["//visibility:public"],
)
cc_test(
name = "opencv_video_decoder_calculator_test",
srcs = ["opencv_video_decoder_calculator_test.cc"],
data = ["//mediapipe/calculators/video/testdata:test_videos"],
data = [":test_videos"],
deps = [
":opencv_video_decoder_calculator",
"//mediapipe/framework:calculator_runner",
@@ -151,7 +163,7 @@ cc_test(
cc_test(
name = "opencv_video_encoder_calculator_test",
srcs = ["opencv_video_encoder_calculator_test.cc"],
data = ["//mediapipe/calculators/video/testdata:test_videos"],
data = [":test_videos"],
deps = [
":opencv_video_decoder_calculator",
":opencv_video_encoder_calculator",
@@ -175,7 +187,6 @@ cc_test(
cc_test(
name = "tvl1_optical_flow_calculator_test",
srcs = ["tvl1_optical_flow_calculator_test.cc"],
data = ["//mediapipe/calculators/image/testdata:test_images"],
deps = [
":tvl1_optical_flow_calculator",
"//mediapipe/framework:calculator_framework",
mediapipe/calculators/video/opencv_video_decoder_calculator.cc
+1
View File
@@ -123,6 +123,7 @@ class OpenCvVideoDecoderCalculator : public CalculatorBase {
cc->Outputs()
.Tag("VIDEO_PRESTREAM")
.Add(header.release(), Timestamp::PreStream());
cc->Outputs().Tag("VIDEO_PRESTREAM").Close();
}
// Rewind to the very first frame.
cap_->set(cv::CAP_PROP_POS_AVI_RATIO, 0);
mediapipe/calculators/video/testdata/BUILD
Vendored
-26
View File
@@ -1,26 +0,0 @@
# Copyright 2019 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
licenses(["notice"]) # Apache 2.0
filegroup(
name = "test_videos",
srcs = [
"format_FLV_H264_AAC.video",
"format_MKV_VP8_VORBIS.video",
"format_MP4_AVC720P_AAC.video",
],
visibility = ["//visibility:public"],
)
mediapipe/calculators/video/tool/BUILD
+4 -4
View File
@@ -13,24 +13,24 @@
# See the License for the specific language governing permissions and
# limitations under the License.
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
licenses(["notice"]) # Apache 2.0
package(default_visibility = ["//mediapipe/calculators/video:__subpackages__"])
exports_files(["LICENSE"])
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
proto_library(
name = "flow_quantizer_model_proto",
srcs = ["flow_quantizer_model.proto"],
visibility = ["//mediapipe:__subpackages__"],
visibility = ["//visibility:public"],
)
mediapipe_cc_proto_library(
name = "flow_quantizer_model_cc_proto",
srcs = ["flow_quantizer_model.proto"],
visibility = ["//mediapipe:__subpackages__"],
visibility = ["//visibility:public"],
deps = [":flow_quantizer_model_proto"],
)