talksik/mediapipe
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Project import generated by Copybara.

Browse Source 
GitOrigin-RevId: f9a66589eaf652bb93f8e37ed9e4da26e59ef214
This commit is contained in:
MediaPipe Team
2020-10-19 13:23:25 -04:00
committed by chuoling
parent cccf6244d3
commit c828392681
68 changed files with 890 additions and 487 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mediapipe/calculators/core/BUILD
+9
View File
@@ -225,6 +225,15 @@ cc_library(
name = "concatenate_vector_calculator",
srcs = ["concatenate_vector_calculator.cc"],
hdrs = ["concatenate_vector_calculator.h"],
copts = select({
# Needed for "//mediapipe/framework/formats:tensor" compatibility on Apple
# platforms for Metal pulled in via the tensor.h header.
"//mediapipe:apple": [
"-x objective-c++",
"-fobjc-arc", # enable reference-counting
],
"//conditions:default": [],
}),
visibility = ["//visibility:public"],
deps = [
":concatenate_vector_calculator_cc_proto",
mediapipe/calculators/core/gate_calculator.cc
+36 -73
View File
@@ -59,30 +59,16 @@ std::string ToString(GateState state) {
// ALLOW or DISALLOW can also be specified as an input side packet. The rules
// for evaluation remain the same as above.
//
// If side_input_has_precedence isn't set in the calculator option,
// ALLOW/DISALLOW inputs must be specified either using input stream or
// via input side packet but not both. Otherwise, both input stream and input
// side packet can be specified and the calculator will take one signal over the
// other based on the value of the side_input_has_precedence field.
// via input side packet but not both.
//
// Intended to be used with the default input stream handler, which synchronizes
// all data input streams with the ALLOW/DISALLOW control input stream.
//
// Example configs:
// Example config:
// node {
// calculator: "GateCalculator"
// input_stream: "input_stream0"
// input_stream: "input_stream1"
// input_stream: "input_streamN"
// input_side_packet: "ALLOW:allow" or "DISALLOW:disallow"
// output_stream: "STATE_CHANGE:state_change"
// output_stream: "output_stream0"
// output_stream: "output_stream1"
// output_stream: "output_streamN"
// }
//
// node {
// calculator: "GateCalculator"
// input_stream: "input_stream0"
// input_stream: "input_stream1"
// input_stream: "input_streamN"
@@ -92,25 +78,6 @@ std::string ToString(GateState state) {
// output_stream: "output_stream1"
// output_stream: "output_streamN"
// }
//
// With side_input_has_precedence:
// node {
// calculator: "GateCalculator"
// input_stream: "input_stream0"
// input_stream: "input_stream1"
// input_stream: "input_streamN"
// input_stream: "ALLOW:allow_stream" or "DISALLOW:disallow_stream"
// input_side_packet: "ALLOW:allow_packet" or "DISALLOW:disallow_packet"
// output_stream: "STATE_CHANGE:state_change"
// output_stream: "output_stream0"
// output_stream: "output_stream1"
// output_stream: "output_streamN"
// options: {
// [mediapipe.GateCalculatorOptions.ext] {
// side_input_has_precedence: true or false
// }
// }
// }
class GateCalculator : public CalculatorBase {
public:
GateCalculator() {}
@@ -121,15 +88,9 @@ class GateCalculator : public CalculatorBase {
cc->InputSidePackets().HasTag("DISALLOW");
bool input_via_stream =
cc->Inputs().HasTag("ALLOW") || cc->Inputs().HasTag("DISALLOW");
const auto& options = cc->Options<::mediapipe::GateCalculatorOptions>();
if (options.has_side_input_has_precedence()) {
RET_CHECK(input_via_side_packet && input_via_stream);
} else {
// Only one of input_side_packet or input_stream may specify
// ALLOW/DISALLOW input when side_input_has_precedence is not set
// in the options.
RET_CHECK(input_via_side_packet ^ input_via_stream);
}
// Only one of input_side_packet or input_stream may specify ALLOW/DISALLOW
// input.
RET_CHECK(input_via_side_packet ^ input_via_stream);
if (input_via_side_packet) {
RET_CHECK(cc->InputSidePackets().HasTag("ALLOW") ^
@@ -140,8 +101,7 @@ class GateCalculator : public CalculatorBase {
} else {
cc->InputSidePackets().Tag("DISALLOW").Set<bool>();
}
}
if (input_via_stream) {
} else {
RET_CHECK(cc->Inputs().HasTag("ALLOW") ^ cc->Inputs().HasTag("DISALLOW"));
if (cc->Inputs().HasTag("ALLOW")) {
@@ -174,13 +134,19 @@ class GateCalculator : public CalculatorBase {
}
::mediapipe::Status Open(CalculatorContext* cc) final {
bool use_side_packet_for_allow_disallow = false;
const auto& options = cc->Options<::mediapipe::GateCalculatorOptions>();
use_calculator_option_for_allow_disallow_ =
options.has_allowance_override();
if (use_calculator_option_for_allow_disallow_) {
allow_by_calculator_option_ = options.allowance_override();
}
if (cc->InputSidePackets().HasTag("ALLOW")) {
use_side_packet_for_allow_disallow = true;
use_side_packet_for_allow_disallow_ = true;
allow_by_side_packet_decision_ =
cc->InputSidePackets().Tag("ALLOW").Get<bool>();
} else if (cc->InputSidePackets().HasTag("DISALLOW")) {
use_side_packet_for_allow_disallow = true;
use_side_packet_for_allow_disallow_ = true;
allow_by_side_packet_decision_ =
!cc->InputSidePackets().Tag("DISALLOW").Get<bool>();
}
@@ -190,33 +156,28 @@ class GateCalculator : public CalculatorBase {
last_gate_state_ = GATE_UNINITIALIZED;
RET_CHECK_OK(CopyInputHeadersToOutputs(cc->Inputs(), &cc->Outputs()));
const auto& options = cc->Options<::mediapipe::GateCalculatorOptions>();
empty_packets_as_allow_ = options.empty_packets_as_allow();
if (!options.has_side_input_has_precedence()) {
side_input_has_precedence_ = use_side_packet_for_allow_disallow;
} else {
side_input_has_precedence_ = options.side_input_has_precedence();
}
return ::mediapipe::OkStatus();
}
::mediapipe::Status Process(CalculatorContext* cc) final {
bool allow_by_stream = empty_packets_as_allow_;
if (cc->Inputs().HasTag("ALLOW") && !cc->Inputs().Tag("ALLOW").IsEmpty()) {
allow_by_stream = cc->Inputs().Tag("ALLOW").Get<bool>();
}
if (cc->Inputs().HasTag("DISALLOW") &&
!cc->Inputs().Tag("DISALLOW").IsEmpty()) {
allow_by_stream = !cc->Inputs().Tag("DISALLOW").Get<bool>();
}
const bool allow_by_side_packet =
allow_by_side_packet_decision_ || empty_packets_as_allow_;
bool allow = false;
if (side_input_has_precedence_) {
allow = allow_by_side_packet;
} else {
allow = allow_by_stream;
// The allow/disallow signal in the calculator option has the highest
// priority. If it's not set, use the stream/side packet signal.
bool allow = allow_by_calculator_option_;
if (!use_calculator_option_for_allow_disallow_) {
allow = empty_packets_as_allow_;
if (use_side_packet_for_allow_disallow_) {
allow = allow_by_side_packet_decision_;
} else {
if (cc->Inputs().HasTag("ALLOW") &&
!cc->Inputs().Tag("ALLOW").IsEmpty()) {
allow = cc->Inputs().Tag("ALLOW").Get<bool>();
}
if (cc->Inputs().HasTag("DISALLOW") &&
!cc->Inputs().Tag("DISALLOW").IsEmpty()) {
allow = !cc->Inputs().Tag("DISALLOW").Get<bool>();
}
}
}
const GateState new_gate_state = allow ? GATE_ALLOW : GATE_DISALLOW;
@@ -251,9 +212,11 @@ class GateCalculator : public CalculatorBase {
private:
GateState last_gate_state_ = GATE_UNINITIALIZED;
int num_data_streams_;
bool empty_packets_as_allow_ = false;
bool use_side_packet_for_allow_disallow_ = false;
bool allow_by_side_packet_decision_ = false;
bool empty_packets_as_allow_;
bool side_input_has_precedence_;
bool use_calculator_option_for_allow_disallow_ = false;
bool allow_by_calculator_option_ = false;
};
REGISTER_CALCULATOR(GateCalculator);
mediapipe/calculators/core/gate_calculator.proto
+4 -7
View File
@@ -28,11 +28,8 @@ message GateCalculatorOptions {
// this option to true inverts that, allowing the data packets to go through.
optional bool empty_packets_as_allow = 1;
// Input side packet and input stream are allowed to coexist only if this
// field is set. When it's set to true, the input side packet has higher
// precedence and the input stream signal will be ignored. When it's set to
// false, the input stream signal always overrides the input side packet
// signal.
//
optional bool side_input_has_precedence = 2;
// If set, the calculator will always allow (if set to yes) or disallow (if
// set to no) the input streams to pass through, and ignore the ALLOW or
// DISALLOW input stream or side input packets.
optional bool allowance_override = 2;
}
mediapipe/calculators/core/gate_calculator_test.cc
+11 -8
View File
@@ -330,45 +330,48 @@ TEST_F(GateCalculatorTest, AllowInitialNoStateTransition) {
ASSERT_EQ(0, output.size());
}
TEST_F(GateCalculatorTest, TestOverrideDecisionBySidePacketSignal) {
TEST_F(GateCalculatorTest,
TestCalculatorOptionDecisionOverrideOverStreamSingal) {
SetRunner(R"(
calculator: "GateCalculator"
input_stream: "test_input"
input_stream: "ALLOW:gating_stream"
input_side_packet: "ALLOW:gating_packet"
output_stream: "test_output"
options: {
[mediapipe.GateCalculatorOptions.ext] {
side_input_has_precedence: true
allowance_override: false
}
}
)");
constexpr int64 kTimestampValue0 = 42;
runner()->MutableSidePackets()->Tag("ALLOW") = Adopt(new bool(false));
// The CalculatorOptions says disallow and the stream says allow. Should
// follow the CalculatorOptions' decision to disallow outputting anything.
RunTimeStep(kTimestampValue0, "ALLOW", true);
const std::vector<Packet>& output = runner()->Outputs().Get("", 0).packets;
ASSERT_EQ(0, output.size());
}
TEST_F(GateCalculatorTest, TestOverrideDecisionByStreamSignal) {
TEST_F(GateCalculatorTest,
TestCalculatorOptionDecisionOverrideOverSidePacketSingal) {
SetRunner(R"(
calculator: "GateCalculator"
input_stream: "test_input"
input_stream: "ALLOW:gating_stream"
input_side_packet: "ALLOW:gating_packet"
output_stream: "test_output"
options: {
[mediapipe.GateCalculatorOptions.ext] {
side_input_has_precedence: false
allowance_override: true
}
}
)");
constexpr int64 kTimestampValue0 = 42;
// The CalculatorOptions says allow and the side packet says disallow. Should
// follow the CalculatorOptions' decision to allow outputting a packet.
runner()->MutableSidePackets()->Tag("ALLOW") = Adopt(new bool(false));
RunTimeStep(kTimestampValue0, "ALLOW", true);
RunTimeStep(kTimestampValue0, true);
const std::vector<Packet>& output = runner()->Outputs().Get("", 0).packets;
ASSERT_EQ(1, output.size());
mediapipe/calculators/image/BUILD
+34 -103
View File
@@ -12,148 +12,78 @@
# See the License for the specific language governing permissions and
# limitations under the License.
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library", "mediapipe_proto_library")
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library")
licenses(["notice"])
package(default_visibility = ["//visibility:private"])
exports_files(["LICENSE"])
proto_library(
mediapipe_proto_library(
name = "opencv_image_encoder_calculator_proto",
srcs = ["opencv_image_encoder_calculator.proto"],
visibility = ["//visibility:public"],
deps = ["//mediapipe/framework:calculator_proto"],
visibility = [
"//visibility:public",
],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "scale_image_calculator_proto",
srcs = ["scale_image_calculator.proto"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
"//mediapipe/framework/formats:image_format_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "set_alpha_calculator_proto",
srcs = ["set_alpha_calculator.proto"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "image_cropping_calculator_proto",
srcs = ["image_cropping_calculator.proto"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "bilateral_filter_calculator_proto",
srcs = ["bilateral_filter_calculator.proto"],
visibility = [
"//visibility:public",
],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "recolor_calculator_proto",
srcs = ["recolor_calculator.proto"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
"//mediapipe/util:color_proto",
],
)
mediapipe_cc_proto_library(
name = "opencv_image_encoder_calculator_cc_proto",
srcs = ["opencv_image_encoder_calculator.proto"],
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
visibility = [
"//visibility:public",
],
deps = [":opencv_image_encoder_calculator_proto"],
)
mediapipe_cc_proto_library(
name = "opencv_encoded_image_to_image_frame_calculator_cc_proto",
srcs = ["opencv_encoded_image_to_image_frame_calculator.proto"],
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
visibility = ["//visibility:public"],
deps = [":opencv_encoded_image_to_image_frame_calculator_proto"],
)
mediapipe_cc_proto_library(
name = "mask_overlay_calculator_cc_proto",
srcs = ["mask_overlay_calculator.proto"],
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
visibility = ["//visibility:public"],
deps = [":mask_overlay_calculator_proto"],
)
mediapipe_cc_proto_library(
name = "scale_image_calculator_cc_proto",
srcs = ["scale_image_calculator.proto"],
cc_deps = [
"//mediapipe/framework:calculator_cc_proto",
"//mediapipe/framework/formats:image_format_cc_proto",
],
visibility = ["//visibility:public"],
deps = [":scale_image_calculator_proto"],
)
mediapipe_cc_proto_library(
name = "set_alpha_calculator_cc_proto",
srcs = ["set_alpha_calculator.proto"],
cc_deps = [
"//mediapipe/framework:calculator_cc_proto",
],
visibility = ["//visibility:public"],
deps = [":set_alpha_calculator_proto"],
)
mediapipe_cc_proto_library(
name = "image_cropping_calculator_cc_proto",
srcs = ["image_cropping_calculator.proto"],
cc_deps = [
"//mediapipe/framework:calculator_cc_proto",
],
visibility = ["//visibility:public"],
deps = [":image_cropping_calculator_proto"],
)
mediapipe_cc_proto_library(
name = "bilateral_filter_calculator_cc_proto",
srcs = ["bilateral_filter_calculator.proto"],
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
visibility = [
"//visibility:public",
],
deps = [":bilateral_filter_calculator_proto"],
)
mediapipe_cc_proto_library(
name = "recolor_calculator_cc_proto",
srcs = ["recolor_calculator.proto"],
cc_deps = [
"//mediapipe/framework:calculator_cc_proto",
"//mediapipe/util:color_cc_proto",
],
visibility = ["//visibility:public"],
deps = [":recolor_calculator_proto"],
)
cc_library(
name = "color_convert_calculator",
srcs = ["color_convert_calculator.cc"],
@@ -550,32 +480,33 @@ cc_test(
],
)
proto_library(
mediapipe_proto_library(
name = "mask_overlay_calculator_proto",
srcs = ["mask_overlay_calculator.proto"],
visibility = ["//visibility:public"],
deps = ["//mediapipe/framework:calculator_proto"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "opencv_encoded_image_to_image_frame_calculator_proto",
srcs = ["opencv_encoded_image_to_image_frame_calculator.proto"],
visibility = ["//visibility:public"],
deps = ["//mediapipe/framework:calculator_proto"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "feature_detector_calculator_proto",
srcs = ["feature_detector_calculator.proto"],
deps = ["//mediapipe/framework:calculator_proto"],
)
mediapipe_cc_proto_library(
name = "feature_detector_calculator_cc_proto",
srcs = ["feature_detector_calculator.proto"],
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
visibility = ["//visibility:public"],
deps = [":feature_detector_calculator_proto"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
cc_library(
mediapipe/calculators/tensorflow/BUILD
-1
View File
@@ -311,7 +311,6 @@ cc_library(
"@org_tensorflow//tensorflow/core:framework",
],
"//mediapipe:android": [
"@org_tensorflow//tensorflow/core:portable_tensorflow_lib_lite",
],
}),
alwayslink = 1,
mediapipe/calculators/tensorflow/pack_media_sequence_calculator.cc
+30 -6
View File
@@ -184,6 +184,7 @@ class PackMediaSequenceCalculator : public CalculatorBase {
features_present_[tag] = false;
}
replace_keypoints_ = false;
if (cc->Options<PackMediaSequenceCalculatorOptions>()
.replace_data_instead_of_append()) {
for (const auto& tag : cc->Inputs().GetTags()) {
@@ -212,6 +213,15 @@ class PackMediaSequenceCalculator : public CalculatorBase {
}
mpms::ClearBBox(key, sequence_.get());
mpms::ClearBBoxTimestamp(key, sequence_.get());
mpms::ClearBBoxIsAnnotated(key, sequence_.get());
mpms::ClearBBoxNumRegions(key, sequence_.get());
mpms::ClearBBoxLabelString(key, sequence_.get());
mpms::ClearBBoxLabelIndex(key, sequence_.get());
mpms::ClearBBoxClassString(key, sequence_.get());
mpms::ClearBBoxClassIndex(key, sequence_.get());
mpms::ClearBBoxTrackString(key, sequence_.get());
mpms::ClearBBoxTrackIndex(key, sequence_.get());
mpms::ClearUnmodifiedBBoxTimestamp(key, sequence_.get());
}
if (absl::StartsWith(tag, kFloatFeaturePrefixTag)) {
std::string key = tag.substr(sizeof(kFloatFeaturePrefixTag) /
@@ -223,8 +233,7 @@ class PackMediaSequenceCalculator : public CalculatorBase {
if (absl::StartsWith(tag, kKeypointsTag)) {
std::string key =
tag.substr(sizeof(kKeypointsTag) / sizeof(*kKeypointsTag) - 1);
mpms::ClearBBoxPoint(key, sequence_.get());
mpms::ClearBBoxTimestamp(key, sequence_.get());
replace_keypoints_ = true;
}
}
if (cc->Inputs().HasTag(kForwardFlowEncodedTag)) {
@@ -342,11 +351,25 @@ class PackMediaSequenceCalculator : public CalculatorBase {
.Get<std::unordered_map<
std::string, std::vector<std::pair<float, float>>>>();
for (const auto& pair : keypoints) {
mpms::AddBBoxTimestamp(mpms::merge_prefix(key, pair.first),
cc->InputTimestamp().Value(), sequence_.get());
mpms::AddBBoxPoint(mpms::merge_prefix(key, pair.first), pair.second,
sequence_.get());
std::string prefix = mpms::merge_prefix(key, pair.first);
if (replace_keypoints_) {
mpms::ClearBBoxPoint(prefix, sequence_.get());
mpms::ClearBBoxTimestamp(prefix, sequence_.get());
mpms::ClearBBoxIsAnnotated(prefix, sequence_.get());
mpms::ClearBBoxNumRegions(prefix, sequence_.get());
mpms::ClearBBoxLabelString(prefix, sequence_.get());
mpms::ClearBBoxLabelIndex(prefix, sequence_.get());
mpms::ClearBBoxClassString(prefix, sequence_.get());
mpms::ClearBBoxClassIndex(prefix, sequence_.get());
mpms::ClearBBoxTrackString(prefix, sequence_.get());
mpms::ClearBBoxTrackIndex(prefix, sequence_.get());
mpms::ClearUnmodifiedBBoxTimestamp(prefix, sequence_.get());
}
mpms::AddBBoxTimestamp(prefix, cc->InputTimestamp().Value(),
sequence_.get());
mpms::AddBBoxPoint(prefix, pair.second, sequence_.get());
}
replace_keypoints_ = false;
}
if (absl::StartsWith(tag, kFloatContextFeaturePrefixTag) &&
!cc->Inputs().Tag(tag).IsEmpty()) {
@@ -475,6 +498,7 @@ class PackMediaSequenceCalculator : public CalculatorBase {
std::unique_ptr<tf::SequenceExample> sequence_;
std::map<std::string, bool> features_present_;
bool replace_keypoints_;
};
REGISTER_CALCULATOR(PackMediaSequenceCalculator);
mediapipe/calculators/tensorflow/pack_media_sequence_calculator_test.cc
+54
View File
@@ -839,5 +839,59 @@ TEST_F(PackMediaSequenceCalculatorTest, TestReconcilingAnnotations) {
ASSERT_EQ(mpms::GetBBoxTimestampAt("PREFIX", output_sequence, 4), 50);
}
TEST_F(PackMediaSequenceCalculatorTest, TestOverwritingAndReconciling) {
SetUpCalculator({"IMAGE:images", "BBOX:bbox"}, {}, false, true);
auto input_sequence = ::absl::make_unique<tf::SequenceExample>();
cv::Mat image(2, 3, CV_8UC3, cv::Scalar(0, 0, 255));
std::vector<uchar> bytes;
ASSERT_TRUE(cv::imencode(".jpg", image, bytes, {80}));
std::string test_image_string(bytes.begin(), bytes.end());
OpenCvImageEncoderCalculatorResults encoded_image;
encoded_image.set_encoded_image(test_image_string);
int height = 2;
int width = 2;
encoded_image.set_width(width);
encoded_image.set_height(height);
int num_images = 5; // Timestamps: 10, 20, 30, 40, 50
for (int i = 0; i < num_images; ++i) {
auto image_ptr =
::absl::make_unique<OpenCvImageEncoderCalculatorResults>(encoded_image);
runner_->MutableInputs()->Tag("IMAGE").packets.push_back(
Adopt(image_ptr.release()).At(Timestamp(i)));
}
for (int i = 0; i < num_images; ++i) {
auto detections = ::absl::make_unique<::std::vector<Detection>>();
Detection detection;
detection = Detection();
detection.add_label("relative bbox");
detection.add_label_id(1);
detection.add_score(0.75);
Location::CreateRelativeBBoxLocation(0, 0.5, 0.5, 0.5)
.ConvertToProto(detection.mutable_location_data());
detections->push_back(detection);
runner_->MutableInputs()->Tag("BBOX").packets.push_back(
Adopt(detections.release()).At(Timestamp(i)));
}
for (int i = 0; i < 10; ++i) {
mpms::AddBBoxTimestamp(-1, input_sequence.get());
mpms::AddBBoxIsAnnotated(-1, input_sequence.get());
mpms::AddBBoxNumRegions(-1, input_sequence.get());
mpms::AddBBoxLabelString({"anything"}, input_sequence.get());
mpms::AddBBoxLabelIndex({-1}, input_sequence.get());
mpms::AddBBoxClassString({"anything"}, input_sequence.get());
mpms::AddBBoxClassIndex({-1}, input_sequence.get());
mpms::AddBBoxTrackString({"anything"}, input_sequence.get());
mpms::AddBBoxTrackIndex({-1}, input_sequence.get());
}
runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
Adopt(input_sequence.release());
// If the all the previous values aren't cleared, this assert will fail.
MP_ASSERT_OK(runner_->Run());
}
} // namespace
} // namespace mediapipe
mediapipe/calculators/tflite/tflite_inference_calculator.cc
+18 -1
View File
@@ -281,6 +281,9 @@ class TfLiteInferenceCalculator : public CalculatorBase {
bool use_quantized_tensors_ = false;
bool use_advanced_gpu_api_ = false;
bool allow_precision_loss_ = false;
::mediapipe::TfLiteInferenceCalculatorOptions_Delegate_Gpu_API
tflite_gpu_runner_api_;
bool use_kernel_caching_ = false;
std::string cached_kernel_filename_;
@@ -365,6 +368,8 @@ bool ShouldUseGpu(CC* cc) {
options.has_delegate() &&
options.delegate().has_gpu() &&
options.delegate().gpu().use_advanced_gpu_api();
allow_precision_loss_ = options.delegate().gpu().allow_precision_loss();
tflite_gpu_runner_api_ = options.delegate().gpu().api();
use_kernel_caching_ =
use_advanced_gpu_api_ && options.delegate().gpu().use_kernel_caching();
@@ -703,11 +708,23 @@ bool ShouldUseGpu(CC* cc) {
// Create runner
tflite::gpu::InferenceOptions options;
options.priority1 = tflite::gpu::InferencePriority::MIN_LATENCY;
options.priority1 = allow_precision_loss_
? tflite::gpu::InferencePriority::MIN_LATENCY
: tflite::gpu::InferencePriority::MAX_PRECISION;
options.priority2 = tflite::gpu::InferencePriority::AUTO;
options.priority3 = tflite::gpu::InferencePriority::AUTO;
options.usage = tflite::gpu::InferenceUsage::SUSTAINED_SPEED;
tflite_gpu_runner_ = std::make_unique<tflite::gpu::TFLiteGPURunner>(options);
if (tflite_gpu_runner_api_ ==
::mediapipe::TfLiteInferenceCalculatorOptions_Delegate_Gpu_API::
TfLiteInferenceCalculatorOptions_Delegate_Gpu_API_OPENGL) {
tflite_gpu_runner_->ForceOpenGL();
}
if (tflite_gpu_runner_api_ ==
::mediapipe::TfLiteInferenceCalculatorOptions_Delegate_Gpu_API::
TfLiteInferenceCalculatorOptions_Delegate_Gpu_API_OPENCL) {
tflite_gpu_runner_->ForceOpenCL();
}
MP_RETURN_IF_ERROR(
tflite_gpu_runner_->InitializeWithModel(model, op_resolver));
mediapipe/calculators/tflite/tflite_inference_calculator.proto
+14
View File
@@ -49,6 +49,20 @@ message TfLiteInferenceCalculatorOptions {
// delegate: { gpu { use_advanced_gpu_api: true } }
optional bool use_advanced_gpu_api = 1 [default = false];
// This option is valid for TFLite GPU delegate API2 only,
// Choose any of available APIs to force running inference using it.
enum API {
ANY = 0;
OPENGL = 1;
OPENCL = 2;
}
optional API api = 4 [default = ANY];
// This option is valid for TFLite GPU delegate API2 only,
// Set to true to use 16-bit float precision. If max precision is needed,
// set to false for 32-bit float calculations only.
optional bool allow_precision_loss = 3 [default = true];
// Load pre-compiled serialized binary cache to accelerate init process.
// Only available for OpenCL delegate on Android.
optional bool use_kernel_caching = 2 [default = false];
mediapipe/calculators/util/BUILD
+3 -2
View File
@@ -18,8 +18,6 @@ licenses(["notice"])
package(default_visibility = ["//visibility:public"])
exports_files(["LICENSE"])
cc_library(
name = "alignment_points_to_rects_calculator",
srcs = ["alignment_points_to_rects_calculator.cc"],
@@ -250,9 +248,11 @@ cc_library(
"@com_google_absl//absl/strings",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/formats:image_frame_opencv",
"//mediapipe/framework/formats:video_stream_header",
"//mediapipe/framework/port:logging",
"//mediapipe/framework/port:opencv_core",
"//mediapipe/framework/port:opencv_imgproc",
"//mediapipe/framework/port:status",
"//mediapipe/framework/port:vector",
"//mediapipe/util:annotation_renderer",
@@ -276,6 +276,7 @@ cc_library(
deps = [
":detection_label_id_to_text_calculator_cc_proto",
"//mediapipe/framework/formats:detection_cc_proto",
"@com_google_absl//absl/container:node_hash_map",
"//mediapipe/framework/port:status",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:packet",
mediapipe/calculators/util/annotation_overlay_calculator.cc
+72 -75
View File
@@ -20,9 +20,11 @@
#include "mediapipe/framework/calculator_options.pb.h"
#include "mediapipe/framework/formats/image_format.pb.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/formats/image_frame_opencv.h"
#include "mediapipe/framework/formats/video_stream_header.h"
#include "mediapipe/framework/port/logging.h"
#include "mediapipe/framework/port/opencv_core_inc.h"
#include "mediapipe/framework/port/opencv_imgproc_inc.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/framework/port/vector.h"
#include "mediapipe/util/annotation_renderer.h"
@@ -40,13 +42,9 @@ namespace mediapipe {
namespace {
constexpr char kInputFrameTag[] = "IMAGE";
constexpr char kOutputFrameTag[] = "IMAGE";
constexpr char kInputVectorTag[] = "VECTOR";
constexpr char kInputFrameTagGpu[] = "IMAGE_GPU";
constexpr char kOutputFrameTagGpu[] = "IMAGE_GPU";
constexpr char kVectorTag[] = "VECTOR";
constexpr char kGpuBufferTag[] = "IMAGE_GPU";
constexpr char kImageFrameTag[] = "IMAGE";
enum { ATTRIB_VERTEX, ATTRIB_TEXTURE_POSITION, NUM_ATTRIBUTES };
@@ -57,12 +55,15 @@ size_t RoundUp(size_t n, size_t m) { return ((n + m - 1) / m) * m; } // NOLINT
// merges the annotation overlay with the image frame. As a result, drawing in
// this color is not supported and it should be set to something unlikely used.
constexpr uchar kAnnotationBackgroundColor = 2; // Grayscale value.
// Future Image type.
inline bool HasImageTag(mediapipe::CalculatorContext* cc) { return false; }
} // namespace
// A calculator for rendering data on images.
//
// Inputs:
// 1. IMAGE or IMAGE_GPU (optional): An ImageFrame (or GpuBuffer)
// 1. IMAGE or IMAGE_GPU (optional): An ImageFrame (or GpuBuffer),
// containing the input image.
// If output is CPU, and input isn't provided, the renderer creates a
// blank canvas with the width, height and color provided in the options.
@@ -74,7 +75,8 @@ constexpr uchar kAnnotationBackgroundColor = 2; // Grayscale value.
// input vector items. These input streams are tagged with "VECTOR".
//
// Output:
// 1. IMAGE or IMAGE_GPU: A rendered ImageFrame (or GpuBuffer).
// 1. IMAGE or IMAGE_GPU: A rendered ImageFrame (or GpuBuffer),
// Note: Output types should match their corresponding input stream type.
//
// For CPU input frames, only SRGBA, SRGB and GRAY8 format are supported. The
// output format is the same as input except for GRAY8 where the output is in
@@ -133,14 +135,17 @@ class AnnotationOverlayCalculator : public CalculatorBase {
::mediapipe::Status CreateRenderTargetCpu(CalculatorContext* cc,
std::unique_ptr<cv::Mat>& image_mat,
ImageFormat::Format* target_format);
template <typename Type, const char* Tag>
::mediapipe::Status CreateRenderTargetGpu(
CalculatorContext* cc, std::unique_ptr<cv::Mat>& image_mat);
template <typename Type, const char* Tag>
::mediapipe::Status RenderToGpu(CalculatorContext* cc, uchar* overlay_image);
::mediapipe::Status RenderToCpu(CalculatorContext* cc,
const ImageFormat::Format& target_format,
uchar* data_image);
::mediapipe::Status GlRender(CalculatorContext* cc);
template <typename Type, const char* Tag>
::mediapipe::Status GlSetup(CalculatorContext* cc);
// Options for the calculator.
@@ -172,24 +177,26 @@ REGISTER_CALCULATOR(AnnotationOverlayCalculator);
bool use_gpu = false;
if (cc->Inputs().HasTag(kInputFrameTag) &&
cc->Inputs().HasTag(kInputFrameTagGpu)) {
if (cc->Inputs().HasTag(kImageFrameTag) &&
cc->Inputs().HasTag(kGpuBufferTag)) {
return ::mediapipe::InternalError("Cannot have multiple input images.");
}
if (cc->Inputs().HasTag(kInputFrameTagGpu) !=
cc->Outputs().HasTag(kOutputFrameTagGpu)) {
if (cc->Inputs().HasTag(kGpuBufferTag) !=
cc->Outputs().HasTag(kGpuBufferTag)) {
return ::mediapipe::InternalError("GPU output must have GPU input.");
}
// Input image to render onto copy of.
// Input image to render onto copy of. Should be same type as output.
#if !defined(MEDIAPIPE_DISABLE_GPU)
if (cc->Inputs().HasTag(kInputFrameTagGpu)) {
cc->Inputs().Tag(kInputFrameTagGpu).Set<mediapipe::GpuBuffer>();
use_gpu |= true;
if (cc->Inputs().HasTag(kGpuBufferTag)) {
cc->Inputs().Tag(kGpuBufferTag).Set<mediapipe::GpuBuffer>();
CHECK(cc->Outputs().HasTag(kGpuBufferTag));
use_gpu = true;
}
#endif // !MEDIAPIPE_DISABLE_GPU
if (cc->Inputs().HasTag(kInputFrameTag)) {
cc->Inputs().Tag(kInputFrameTag).Set<ImageFrame>();
if (cc->Inputs().HasTag(kImageFrameTag)) {
cc->Inputs().Tag(kImageFrameTag).Set<ImageFrame>();
CHECK(cc->Outputs().HasTag(kImageFrameTag));
}
// Data streams to render.
@@ -197,7 +204,7 @@ REGISTER_CALCULATOR(AnnotationOverlayCalculator);
++id) {
auto tag_and_index = cc->Inputs().TagAndIndexFromId(id);
std::string tag = tag_and_index.first;
if (tag == kInputVectorTag) {
if (tag == kVectorTag) {
cc->Inputs().Get(id).Set<std::vector<RenderData>>();
} else if (tag.empty()) {
// Empty tag defaults to accepting a single object of RenderData type.
@@ -205,15 +212,14 @@ REGISTER_CALCULATOR(AnnotationOverlayCalculator);
}
}
// Rendered image.
// Rendered image. Should be same type as input.
#if !defined(MEDIAPIPE_DISABLE_GPU)
if (cc->Outputs().HasTag(kOutputFrameTagGpu)) {
cc->Outputs().Tag(kOutputFrameTagGpu).Set<mediapipe::GpuBuffer>();
use_gpu |= true;
if (cc->Outputs().HasTag(kGpuBufferTag)) {
cc->Outputs().Tag(kGpuBufferTag).Set<mediapipe::GpuBuffer>();
}
#endif // !MEDIAPIPE_DISABLE_GPU
if (cc->Outputs().HasTag(kOutputFrameTag)) {
cc->Outputs().Tag(kOutputFrameTag).Set<ImageFrame>();
if (cc->Outputs().HasTag(kImageFrameTag)) {
cc->Outputs().Tag(kImageFrameTag).Set<ImageFrame>();
}
if (use_gpu) {
@@ -229,20 +235,16 @@ REGISTER_CALCULATOR(AnnotationOverlayCalculator);
cc->SetOffset(TimestampDiff(0));
options_ = cc->Options<AnnotationOverlayCalculatorOptions>();
if (cc->Inputs().HasTag(kInputFrameTagGpu) &&
cc->Outputs().HasTag(kOutputFrameTagGpu)) {
if (cc->Inputs().HasTag(kGpuBufferTag) || HasImageTag(cc)) {
#if !defined(MEDIAPIPE_DISABLE_GPU)
use_gpu_ = true;
#else
RET_CHECK_FAIL() << "GPU processing not enabled.";
#endif // !MEDIAPIPE_DISABLE_GPU
}
if (cc->Inputs().HasTag(kInputFrameTagGpu) ||
cc->Inputs().HasTag(kInputFrameTag)) {
if (cc->Inputs().HasTag(kGpuBufferTag) ||
cc->Inputs().HasTag(kImageFrameTag) || HasImageTag(cc)) {
image_frame_available_ = true;
} else {
image_frame_available_ = false;
RET_CHECK(options_.has_canvas_width_px());
RET_CHECK(options_.has_canvas_height_px());
}
@@ -253,14 +255,12 @@ REGISTER_CALCULATOR(AnnotationOverlayCalculator);
if (use_gpu_) renderer_->SetScaleFactor(options_.gpu_scale_factor());
// Set the output header based on the input header (if present).
const char* input_tag = use_gpu_ ? kInputFrameTagGpu : kInputFrameTag;
const char* output_tag = use_gpu_ ? kOutputFrameTagGpu : kOutputFrameTag;
if (image_frame_available_ &&
!cc->Inputs().Tag(input_tag).Header().IsEmpty()) {
const char* tag = use_gpu_ ? kGpuBufferTag : kImageFrameTag;
if (image_frame_available_ && !cc->Inputs().Tag(tag).Header().IsEmpty()) {
const auto& input_header =
cc->Inputs().Tag(input_tag).Header().Get<VideoHeader>();
cc->Inputs().Tag(tag).Header().Get<VideoHeader>();
auto* output_video_header = new VideoHeader(input_header);
cc->Outputs().Tag(output_tag).SetHeader(Adopt(output_video_header));
cc->Outputs().Tag(tag).SetHeader(Adopt(output_video_header));
}
if (use_gpu_) {
@@ -282,15 +282,20 @@ REGISTER_CALCULATOR(AnnotationOverlayCalculator);
if (!gpu_initialized_) {
MP_RETURN_IF_ERROR(
gpu_helper_.RunInGlContext([this, cc]() -> ::mediapipe::Status {
MP_RETURN_IF_ERROR(GlSetup(cc));
return ::mediapipe::OkStatus();
return GlSetup<mediapipe::GpuBuffer, kGpuBufferTag>(cc);
}));
gpu_initialized_ = true;
}
if (cc->Inputs().HasTag(kGpuBufferTag)) {
MP_RETURN_IF_ERROR(
(CreateRenderTargetGpu<mediapipe::GpuBuffer, kGpuBufferTag>(
cc, image_mat)));
}
#endif // !MEDIAPIPE_DISABLE_GPU
MP_RETURN_IF_ERROR(CreateRenderTargetGpu(cc, image_mat));
} else {
MP_RETURN_IF_ERROR(CreateRenderTargetCpu(cc, image_mat, &target_format));
if (cc->Inputs().HasTag(kImageFrameTag)) {
MP_RETURN_IF_ERROR(CreateRenderTargetCpu(cc, image_mat, &target_format));
}
}
// Reset the renderer with the image_mat. No copy here.
@@ -301,7 +306,7 @@ REGISTER_CALCULATOR(AnnotationOverlayCalculator);
++id) {
auto tag_and_index = cc->Inputs().TagAndIndexFromId(id);
std::string tag = tag_and_index.first;
if (!tag.empty() && tag != kInputVectorTag) {
if (!tag.empty() && tag != kVectorTag) {
continue;
}
if (cc->Inputs().Get(id).IsEmpty()) {
@@ -312,7 +317,7 @@ REGISTER_CALCULATOR(AnnotationOverlayCalculator);
const RenderData& render_data = cc->Inputs().Get(id).Get<RenderData>();
renderer_->RenderDataOnImage(render_data);
} else {
RET_CHECK_EQ(kInputVectorTag, tag);
RET_CHECK_EQ(kVectorTag, tag);
const std::vector<RenderData>& render_data_vec =
cc->Inputs().Get(id).Get<std::vector<RenderData>>();
for (const RenderData& render_data : render_data_vec) {
@@ -327,8 +332,8 @@ REGISTER_CALCULATOR(AnnotationOverlayCalculator);
uchar* image_mat_ptr = image_mat->data;
MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext(
[this, cc, image_mat_ptr]() -> ::mediapipe::Status {
MP_RETURN_IF_ERROR(RenderToGpu(cc, image_mat_ptr));
return ::mediapipe::OkStatus();
return RenderToGpu<mediapipe::GpuBuffer, kGpuBufferTag>(
cc, image_mat_ptr);
}));
#endif // !MEDIAPIPE_DISABLE_GPU
} else {
@@ -369,19 +374,21 @@ REGISTER_CALCULATOR(AnnotationOverlayCalculator);
ImageFrame::kDefaultAlignmentBoundary);
#endif // !MEDIAPIPE_DISABLE_GPU
cc->Outputs()
.Tag(kOutputFrameTag)
.Add(output_frame.release(), cc->InputTimestamp());
if (cc->Outputs().HasTag(kImageFrameTag)) {
cc->Outputs()
.Tag(kImageFrameTag)
.Add(output_frame.release(), cc->InputTimestamp());
}
return ::mediapipe::OkStatus();
}
template <typename Type, const char* Tag>
::mediapipe::Status AnnotationOverlayCalculator::RenderToGpu(
CalculatorContext* cc, uchar* overlay_image) {
#if !defined(MEDIAPIPE_DISABLE_GPU)
// Source and destination textures.
const auto& input_frame =
cc->Inputs().Tag(kInputFrameTagGpu).Get<mediapipe::GpuBuffer>();
const auto& input_frame = cc->Inputs().Tag(Tag).Get<Type>();
auto input_texture = gpu_helper_.CreateSourceTexture(input_frame);
auto output_texture = gpu_helper_.CreateDestinationTexture(
@@ -414,10 +421,8 @@ REGISTER_CALCULATOR(AnnotationOverlayCalculator);
}
// Send out blended image as GPU packet.
auto output_frame = output_texture.GetFrame<mediapipe::GpuBuffer>();
cc->Outputs()
.Tag(kOutputFrameTagGpu)
.Add(output_frame.release(), cc->InputTimestamp());
auto output_frame = output_texture.GetFrame<Type>();
cc->Outputs().Tag(Tag).Add(output_frame.release(), cc->InputTimestamp());
// Cleanup
input_texture.Release();
@@ -432,7 +437,7 @@ REGISTER_CALCULATOR(AnnotationOverlayCalculator);
ImageFormat::Format* target_format) {
if (image_frame_available_) {
const auto& input_frame =
cc->Inputs().Tag(kInputFrameTag).Get<ImageFrame>();
cc->Inputs().Tag(kImageFrameTag).Get<ImageFrame>();
int target_mat_type;
switch (input_frame.Format()) {
@@ -455,21 +460,14 @@ REGISTER_CALCULATOR(AnnotationOverlayCalculator);
image_mat = absl::make_unique<cv::Mat>(
input_frame.Height(), input_frame.Width(), target_mat_type);
auto input_mat = formats::MatView(&input_frame);
if (input_frame.Format() == ImageFormat::GRAY8) {
const int target_num_channels =
ImageFrame::NumberOfChannelsForFormat(*target_format);
for (int i = 0; i < input_frame.PixelDataSize(); i++) {
const auto& pix = input_frame.PixelData()[i];
for (int c = 0; c < target_num_channels; c++) {
image_mat->data[i * target_num_channels + c] = pix;
}
}
cv::Mat rgb_mat;
cv::cvtColor(input_mat, rgb_mat, CV_GRAY2RGB);
rgb_mat.copyTo(*image_mat);
} else {
// Make of a copy since the input frame may be consumed by other nodes.
const int buffer_size =
input_frame.Height() * input_frame.Width() *
ImageFrame::NumberOfChannelsForFormat(*target_format);
input_frame.CopyToBuffer(image_mat->data, buffer_size);
input_mat.copyTo(*image_mat);
}
} else {
image_mat = absl::make_unique<cv::Mat>(
@@ -482,13 +480,12 @@ REGISTER_CALCULATOR(AnnotationOverlayCalculator);
return ::mediapipe::OkStatus();
}
template <typename Type, const char* Tag>
::mediapipe::Status AnnotationOverlayCalculator::CreateRenderTargetGpu(
CalculatorContext* cc, std::unique_ptr<cv::Mat>& image_mat) {
#if !defined(MEDIAPIPE_DISABLE_GPU)
if (image_frame_available_) {
const auto& input_frame =
cc->Inputs().Tag(kInputFrameTagGpu).Get<mediapipe::GpuBuffer>();
const auto& input_frame = cc->Inputs().Tag(Tag).Get<Type>();
const mediapipe::ImageFormat::Format format =
mediapipe::ImageFormatForGpuBufferFormat(input_frame.format());
if (format != mediapipe::ImageFormat::SRGBA &&
@@ -564,6 +561,7 @@ REGISTER_CALCULATOR(AnnotationOverlayCalculator);
return ::mediapipe::OkStatus();
}
template <typename Type, const char* Tag>
::mediapipe::Status AnnotationOverlayCalculator::GlSetup(
CalculatorContext* cc) {
#if !defined(MEDIAPIPE_DISABLE_GPU)
@@ -639,8 +637,7 @@ REGISTER_CALCULATOR(AnnotationOverlayCalculator);
const float alignment = ImageFrame::kGlDefaultAlignmentBoundary;
const float scale_factor = options_.gpu_scale_factor();
if (image_frame_available_) {
const auto& input_frame =
cc->Inputs().Tag(kInputFrameTagGpu).Get<mediapipe::GpuBuffer>();
const auto& input_frame = cc->Inputs().Tag(Tag).Get<Type>();
width_ = RoundUp(input_frame.width(), alignment);
height_ = RoundUp(input_frame.height(), alignment);
} else {
mediapipe/calculators/util/detection_label_id_to_text_calculator.cc
+2 -1
View File
@@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include "absl/container/node_hash_map.h"
#include "mediapipe/calculators/util/detection_label_id_to_text_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/detection.pb.h"
@@ -52,7 +53,7 @@ class DetectionLabelIdToTextCalculator : public CalculatorBase {
::mediapipe::Status Process(CalculatorContext* cc) override;
private:
std::unordered_map<int, std::string> label_map_;
absl::node_hash_map<int, std::string> label_map_;
};
REGISTER_CALCULATOR(DetectionLabelIdToTextCalculator);
mediapipe/calculators/video/BUILD
+1
View File
@@ -317,6 +317,7 @@ cc_library(
"//mediapipe/util/tracking:box_tracker",
"//mediapipe/util/tracking:tracking_visualization_utilities",
"@com_google_absl//absl/container:flat_hash_set",
"@com_google_absl//absl/container:node_hash_map",
"@com_google_absl//absl/container:node_hash_set",
"@com_google_absl//absl/strings",
],
mediapipe/calculators/video/box_tracker_calculator.cc
+2 -1
View File
@@ -19,6 +19,7 @@
#include <unordered_set>
#include "absl/container/flat_hash_set.h"
#include "absl/container/node_hash_map.h"
#include "absl/container/node_hash_set.h"
#include "absl/strings/numbers.h"
#include "mediapipe/calculators/video/box_tracker_calculator.pb.h"
@@ -207,7 +208,7 @@ class BoxTrackerCalculator : public CalculatorBase {
// Boxes that are tracked in streaming mode.
MotionBoxMap streaming_motion_boxes_;
std::unordered_map<int, std::pair<TimedBox, TimedBox>> last_tracked_boxes_;
absl::node_hash_map<int, std::pair<TimedBox, TimedBox>> last_tracked_boxes_;
int frame_num_since_reset_ = 0;
// Cache used during streaming mode for fast forward tracking.
mediapipe/calculators/video/tool/BUILD
-2
View File
@@ -19,8 +19,6 @@ licenses(["notice"])
package(default_visibility = ["//mediapipe/calculators/video:__subpackages__"])
exports_files(["LICENSE"])
proto_library(
name = "flow_quantizer_model_proto",
srcs = ["flow_quantizer_model.proto"],
mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/MainActivity.java
+3
View File
@@ -134,6 +134,9 @@ public class MainActivity extends AppCompatActivity {
protected void onPause() {
super.onPause();
converter.close();
// Hide preview display until we re-open the camera again.
previewDisplayView.setVisibility(View.GONE);
}
@Override
mediapipe/examples/android/src/java/com/google/mediapipe/apps/iristrackinggpu/MainActivity.java
+12 -6
View File
@@ -32,16 +32,22 @@ public class MainActivity extends com.google.mediapipe.apps.basic.MainActivity {
private static final String FOCAL_LENGTH_STREAM_NAME = "focal_length_pixel";
private static final String OUTPUT_LANDMARKS_STREAM_NAME = "face_landmarks_with_iris";
private boolean haveAddedSidePackets = false;
@Override
protected void onCameraStarted(SurfaceTexture surfaceTexture) {
super.onCameraStarted(surfaceTexture);
float focalLength = cameraHelper.getFocalLengthPixels();
if (focalLength != Float.MIN_VALUE) {
Packet focalLengthSidePacket = processor.getPacketCreator().createFloat32(focalLength);
Map<String, Packet> inputSidePackets = new HashMap<>();
inputSidePackets.put(FOCAL_LENGTH_STREAM_NAME, focalLengthSidePacket);
processor.setInputSidePackets(inputSidePackets);
// onCameraStarted gets called each time the activity resumes, but we only want to do this once.
if (!haveAddedSidePackets) {
float focalLength = cameraHelper.getFocalLengthPixels();
if (focalLength != Float.MIN_VALUE) {
Packet focalLengthSidePacket = processor.getPacketCreator().createFloat32(focalLength);
Map<String, Packet> inputSidePackets = new HashMap<>();
inputSidePackets.put(FOCAL_LENGTH_STREAM_NAME, focalLengthSidePacket);
processor.setInputSidePackets(inputSidePackets);
}
haveAddedSidePackets = true;
}
}
mediapipe/examples/coral/Dockerfile
+1 -1
View File
@@ -62,7 +62,7 @@ COPY . /mediapipe/
# Install bazel
# Please match the current MediaPipe Bazel requirements according to docs.
ARG BAZEL_VERSION=2.0.0
ARG BAZEL_VERSION=3.4.1
RUN mkdir /bazel && \
wget --no-check-certificate -O /bazel/installer.sh "https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh" && \
wget --no-check-certificate -O /bazel/LICENSE.txt "https://raw.githubusercontent.com/bazelbuild/bazel/master/LICENSE" && \
mediapipe/framework/BUILD
+13
View File
@@ -1107,6 +1107,19 @@ cc_library(
alwayslink = 1,
)
cc_library(
name = "basic_types_registration",
srcs = ["basic_types_registration.cc"],
visibility = ["//visibility:public"],
deps = [
":type_map",
"//mediapipe/framework/port:integral_types",
"@com_google_absl//absl/memory",
"@com_google_absl//absl/strings",
],
alwayslink = 1,
)
cc_library(
name = "validated_graph_config",
srcs = ["validated_graph_config.cc"],
mediapipe/framework/basic_types_registration.cc
+28
View File
@@ -0,0 +1,28 @@
#include <string>
#include <vector>
#include "mediapipe/framework/port/integral_types.h"
#include "mediapipe/framework/type_map.h"
#define MEDIAPIPE_REGISTER_GENERIC_TYPE(type) \
MEDIAPIPE_REGISTER_TYPE( \
::mediapipe::type_map_internal::ReflectType<void(type*)>::Type, #type, \
nullptr, nullptr)
// Note: we cannot define a type which type hash id is already in the map.
// E.g. if tool::GetTypeHash<int>() == tool::GetTypeHash<int32>(), then only one
// can be registered.
MEDIAPIPE_REGISTER_GENERIC_TYPE(bool);
MEDIAPIPE_REGISTER_GENERIC_TYPE(double);
MEDIAPIPE_REGISTER_GENERIC_TYPE(float);
MEDIAPIPE_REGISTER_GENERIC_TYPE(int);
MEDIAPIPE_REGISTER_GENERIC_TYPE(int64);
MEDIAPIPE_REGISTER_GENERIC_TYPE(uint64);
MEDIAPIPE_REGISTER_GENERIC_TYPE(::std::vector<bool>);
MEDIAPIPE_REGISTER_GENERIC_TYPE(::std::vector<double>);
MEDIAPIPE_REGISTER_GENERIC_TYPE(::std::vector<float>);
MEDIAPIPE_REGISTER_GENERIC_TYPE(::std::vector<int>);
MEDIAPIPE_REGISTER_GENERIC_TYPE(::std::vector<int64>);
MEDIAPIPE_REGISTER_GENERIC_TYPE(::std::vector<std::string>);
MEDIAPIPE_REGISTER_GENERIC_TYPE(::std::vector<::std::vector<float>>);
mediapipe/framework/collection_test.cc
+23 -20
View File
@@ -218,10 +218,11 @@ template <typename T>
EXPECT_EQ(values[5], collection_ptr->Get("TAG_C", 0));
// Test const-ness.
EXPECT_EQ(false, std::is_const<typename std::remove_reference<decltype(
collection.Get("TAG_A", 0))>::type>::value);
EXPECT_EQ(true, std::is_const<typename std::remove_reference<decltype(
collection_ptr->Get("TAG_A", 0))>::type>::value);
EXPECT_EQ(false, std::is_const<typename std::remove_reference<
decltype(collection.Get("TAG_A", 0))>::type>::value);
EXPECT_EQ(true,
std::is_const<typename std::remove_reference<
decltype(collection_ptr->Get("TAG_A", 0))>::type>::value);
// Test access using a range based for.
int i = 0;
@@ -278,10 +279,11 @@ template <typename T>
EXPECT_EQ(values[5], collection_ptr->Get("TAG_C", 0));
// Test const-ness.
EXPECT_EQ(false, std::is_const<typename std::remove_reference<decltype(
collection.Get("TAG_A", 0))>::type>::value);
EXPECT_EQ(true, std::is_const<typename std::remove_reference<decltype(
collection_ptr->Get("TAG_A", 0))>::type>::value);
EXPECT_EQ(false, std::is_const<typename std::remove_reference<
decltype(collection.Get("TAG_A", 0))>::type>::value);
EXPECT_EQ(true,
std::is_const<typename std::remove_reference<
decltype(collection_ptr->Get("TAG_A", 0))>::type>::value);
// Test access using a range based for.
int i = 0;
@@ -309,10 +311,10 @@ template <typename T>
// storage == kStoreValue.
EXPECT_EQ(&values[i], collection_ptr->GetPtr(id));
EXPECT_EQ(values[i], *collection_ptr->GetPtr(id));
EXPECT_EQ(false, std::is_const<typename std::remove_reference<decltype(
*collection.GetPtr(id))>::type>::value);
EXPECT_EQ(true, std::is_const<typename std::remove_reference<decltype(
*collection_ptr->GetPtr(id))>::type>::value);
EXPECT_EQ(false, std::is_const<typename std::remove_reference<
decltype(*collection.GetPtr(id))>::type>::value);
EXPECT_EQ(true, std::is_const<typename std::remove_reference<
decltype(*collection_ptr->GetPtr(id))>::type>::value);
++i;
}
@@ -386,10 +388,11 @@ template <typename T>
EXPECT_EQ(&values[5], collection_ptr->Get("TAG_C", 0));
// Test const-ness.
EXPECT_EQ(false, std::is_const<typename std::remove_reference<decltype(
collection.Get("TAG_A", 0))>::type>::value);
EXPECT_EQ(true, std::is_const<typename std::remove_reference<decltype(
collection_ptr->Get("TAG_A", 0))>::type>::value);
EXPECT_EQ(false, std::is_const<typename std::remove_reference<
decltype(collection.Get("TAG_A", 0))>::type>::value);
EXPECT_EQ(true,
std::is_const<typename std::remove_reference<
decltype(collection_ptr->Get("TAG_A", 0))>::type>::value);
// Test access using a range based for.
int i = 0;
@@ -473,8 +476,8 @@ TEST(CollectionTest, TestIteratorFunctions) {
collection.GetPtr(collection.GetId("TAG_B", 1)) = &values[4];
collection.GetPtr(collection.GetId("TAG_C", 0)) = &values[5];
EXPECT_EQ(false, std::is_const<typename std::remove_reference<decltype(
collection.begin())>::type>::value);
EXPECT_EQ(false, std::is_const<typename std::remove_reference<
decltype(collection.begin())>::type>::value);
EXPECT_EQ(values[0], *collection.begin());
EXPECT_EQ(false, collection.begin()->empty());
EXPECT_EQ(false, (*collection.begin()).empty());
@@ -483,8 +486,8 @@ TEST(CollectionTest, TestIteratorFunctions) {
const auto* collection_ptr = &collection;
EXPECT_EQ(true, std::is_const<typename std::remove_reference<decltype(
*collection_ptr->begin())>::type>::value);
EXPECT_EQ(true, std::is_const<typename std::remove_reference<
decltype(*collection_ptr->begin())>::type>::value);
EXPECT_EQ(values[0], *collection_ptr->begin());
EXPECT_EQ(false, collection_ptr->begin()->empty());
EXPECT_EQ(false, (*collection_ptr->begin()).empty());
mediapipe/framework/deps/file_helpers.cc
+1
View File
@@ -16,6 +16,7 @@
#ifdef _WIN32
#include <Windows.h>
#include <direct.h>
#else
#include <dirent.h>
#endif // _WIN32
mediapipe/framework/formats/BUILD
+25 -2
View File
@@ -23,8 +23,6 @@ package(
licenses(["notice"])
exports_files(["LICENSE"])
mediapipe_proto_library(
name = "detection_proto",
srcs = ["detection.proto"],
@@ -32,6 +30,18 @@ mediapipe_proto_library(
deps = ["//mediapipe/framework/formats:location_data_proto"],
)
mediapipe_register_type(
base_name = "detection",
include_headers = ["mediapipe/framework/formats/detection.pb.h"],
types = [
"::mediapipe::Detection",
"::mediapipe::DetectionList",
"::std::vector<::mediapipe::Detection>",
"::std::vector<::mediapipe::DetectionList>",
],
deps = ["//mediapipe/framework/formats:detection_cc_proto"],
)
mediapipe_proto_library(
name = "classification_proto",
srcs = ["classification.proto"],
@@ -214,6 +224,18 @@ mediapipe_proto_library(
deps = ["//mediapipe/framework/formats:location_data_proto"],
)
mediapipe_register_type(
base_name = "rect",
include_headers = ["mediapipe/framework/formats/rect.pb.h"],
types = [
"::mediapipe::Rect",
"::mediapipe::NormalizedRect",
"::std::vector<::mediapipe::Rect>",
"::std::vector<::mediapipe::NormalizedRect>",
],
deps = [":rect_cc_proto"],
)
mediapipe_proto_library(
name = "landmark_proto",
srcs = ["landmark.proto"],
@@ -231,6 +253,7 @@ mediapipe_register_type(
"::mediapipe::NormalizedLandmarkList",
"::std::vector<::mediapipe::Landmark>",
"::std::vector<::mediapipe::NormalizedLandmark>",
"::std::vector<::mediapipe::NormalizedLandmarkList>",
],
deps = [":landmark_cc_proto"],
)
mediapipe/framework/formats/annotation/BUILD
-2
View File
@@ -20,8 +20,6 @@ package(default_visibility = ["//visibility:private"])
licenses(["notice"])
exports_files(["LICENSE"])
mediapipe_proto_library(
name = "locus_proto",
srcs = ["locus.proto"],
mediapipe/framework/formats/motion/BUILD
-2
View File
@@ -18,8 +18,6 @@
licenses(["notice"])
exports_files(["LICENSE"])
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
package(default_visibility = ["//visibility:private"])
mediapipe/framework/input_stream_manager.cc
+4 -3
View File
@@ -163,9 +163,10 @@ template <typename Container>
}
queue_became_full = (!was_queue_full && max_queue_size_ != -1 &&
queue_.size() >= max_queue_size_);
VLOG_IF(3, queue_.size() > 1)
<< "Queue size greater than 1: stream name: " << name_
<< " queue_size: " << queue_.size();
if (queue_.size() > 1) {
VLOG(3) << "Queue size greater than 1: stream name: " << name_
<< " queue_size: " << queue_.size();
}
VLOG(3) << "Input stream:" << name_
<< " becomes non-empty status:" << queue_became_non_empty
<< " Size: " << queue_.size();
mediapipe/framework/legacy_calculator_support.h
+2
View File
@@ -66,6 +66,7 @@ class LegacyCalculatorSupport {
};
};
#if !defined(_MSC_VER)
// We only declare this variable for two specializations of the template because
// it is only meant to be used for these two types.
// Note that, since these variables are members of specific template
@@ -79,6 +80,7 @@ thread_local CalculatorContext*
template <>
thread_local CalculatorContract*
LegacyCalculatorSupport::Scoped<CalculatorContract>::current_;
#endif
} // namespace mediapipe
mediapipe/framework/packet.h
+6 -1
View File
@@ -439,13 +439,18 @@ struct is_concrete_proto_t
template <typename T>
struct MessageRegistrationImpl {
static NoDestructor<mediapipe::RegistrationToken> registration;
// This could have been a lambda inside registration's initializer below, but
// MSVC has a bug with lambdas, so we put it here as a workaround.
static std::unique_ptr<Holder<T>> CreateMessageHolder() {
return absl::make_unique<Holder<T>>(new T);
}
};
// Static members of template classes can be defined in the header.
template <typename T>
NoDestructor<mediapipe::RegistrationToken>
MessageRegistrationImpl<T>::registration(MessageHolderRegistry::Register(
T{}.GetTypeName(), [] { return absl::make_unique<Holder<T>>(new T); }));
T{}.GetTypeName(), MessageRegistrationImpl<T>::CreateMessageHolder));
// For non-Message payloads, this does nothing.
template <typename T, typename Enable = void>
mediapipe/framework/profiler/BUILD
-1
View File
@@ -11,7 +11,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
licenses(["notice"])
mediapipe/framework/profiler/graph_profiler.cc
+4 -2
View File
@@ -204,6 +204,10 @@ void GraphProfiler::Reset() {
Resume();
if (is_tracing_ && IsTraceIntervalEnabled(profiler_config_, tracer()) &&
executor != nullptr) {
// Inform the user via logging the path to the trace logs.
ASSIGN_OR_RETURN(std::string trace_log_path, GetTraceLogPath());
LOG(INFO) << "trace_log_path: " << trace_log_path;
is_running_ = true;
executor->Schedule([this] {
absl::Time deadline = clock_->TimeNow() + tracer()->GetTraceLogInterval();
@@ -583,8 +587,6 @@ void AssignNodeNames(GraphProfile* profile) {
return ::mediapipe::OkStatus();
}
ASSIGN_OR_RETURN(std::string trace_log_path, GetTraceLogPath());
// Inform the user via logging the path to the trace logs.
LOG(INFO) << "trace_log_path: " << trace_log_path;
int log_interval_count = GetLogIntervalCount(profiler_config_);
int log_file_count = GetLogFileCount(profiler_config_);
mediapipe/framework/subgraph.h
+8
View File
@@ -49,6 +49,14 @@ class Subgraph {
static T GetOptions(const Subgraph::SubgraphOptions& supgraph_options) {
return tool::OptionsMap().Initialize(supgraph_options).Get<T>();
}
// Returns the CalculatorGraphConfig::Node specifying the subgraph.
// This provides to Subgraphs the same graph information that GetContract
// provides to Calculators.
static CalculatorGraphConfig::Node GetNode(
const Subgraph::SubgraphOptions& supgraph_options) {
return supgraph_options;
}
};
using SubgraphRegistry = GlobalFactoryRegistry<std::unique_ptr<Subgraph>>;
mediapipe/framework/testdata/BUILD
Vendored
-2
View File
@@ -17,8 +17,6 @@ licenses(["notice"])
package(default_visibility = ["//visibility:private"])
exports_files(["LICENSE"])
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
proto_library(
mediapipe/framework/tool/BUILD
+10 -47
View File
@@ -13,10 +13,7 @@
# limitations under the License.
#
load(
"//mediapipe/framework/port:build_config.bzl",
"mediapipe_cc_proto_library",
)
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library")
load(
"//mediapipe/framework/tool:mediapipe_graph.bzl",
"data_as_c_string",
@@ -44,9 +41,10 @@ cc_library(
],
)
proto_library(
mediapipe_proto_library(
name = "calculator_graph_template_proto",
srcs = ["calculator_graph_template.proto"],
def_py_proto = False,
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_options_proto",
@@ -55,43 +53,14 @@ proto_library(
],
)
java_proto_library(
name = "calculator_graph_template_java_proto",
visibility = ["//visibility:public"],
deps = [":calculator_graph_template_proto"],
)
java_lite_proto_library(
name = "calculator_graph_template_java_proto_lite",
strict_deps = 0,
visibility = ["//visibility:public"],
deps = [":calculator_graph_template_proto"],
)
proto_library(
mediapipe_proto_library(
name = "source_proto",
srcs = ["source.proto"],
visibility = ["//visibility:public"],
deps = ["//mediapipe/framework:calculator_proto"],
)
mediapipe_cc_proto_library(
name = "calculator_graph_template_cc_proto",
srcs = ["calculator_graph_template.proto"],
cc_deps = [
"//mediapipe/framework:calculator_cc_proto",
"//mediapipe/framework/deps:proto_descriptor_cc_proto",
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
visibility = ["//visibility:public"],
deps = [":calculator_graph_template_proto"],
)
mediapipe_cc_proto_library(
name = "source_cc_proto",
srcs = ["source.proto"],
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
visibility = ["//visibility:public"],
deps = [":source_proto"],
)
cc_binary(
@@ -551,22 +520,16 @@ data_as_c_string(
outs = ["test_binarypb.inc"],
)
proto_library(
mediapipe_proto_library(
name = "node_chain_subgraph_proto",
srcs = ["node_chain_subgraph.proto"],
visibility = ["//mediapipe:__subpackages__"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
mediapipe_cc_proto_library(
name = "node_chain_subgraph_cc_proto",
srcs = ["node_chain_subgraph.proto"],
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
visibility = ["//mediapipe:__subpackages__"],
deps = [":node_chain_subgraph_proto"],
)
cc_test(
name = "data_as_c_string_test",
srcs = [
mediapipe/framework/tool/gate_subgraph.proto
+22
View File
@@ -0,0 +1,22 @@
syntax = "proto2";
package mediapipe;
import "mediapipe/framework/calculator.proto";
option java_package = "com.google.mediapipe.proto";
option java_outer_classname = "GateSubgraphProto";
// Options for a gate-subgraph directing traffic to one of several contained
// CalculatorGraphConfig's.
message GateSubgraphOptions {
extend mediapipe.CalculatorOptions {
optional GateSubgraphOptions ext = 297196839;
}
// The contained literal subgraph configuration(s).
repeated CalculatorGraphConfig contained_graph = 1;
// The contained registered subgraphs or calculators.
repeated CalculatorGraphConfig.Node contained_node = 2;
}
mediapipe/framework/tool/subgraph_expansion.cc
+20
View File
@@ -316,5 +316,25 @@ static ::mediapipe::Status PrefixNames(std::string prefix,
return ::mediapipe::OkStatus();
}
CalculatorGraphConfig MakeSingleNodeGraph(CalculatorGraphConfig::Node node) {
using RepeatedStringField = proto_ns::RepeatedPtrField<ProtoString>;
struct Connections {
const RepeatedStringField& node_conns;
RepeatedStringField* graph_conns;
};
CalculatorGraphConfig config;
for (const Connections& item : std::vector<Connections>{
{node.input_stream(), config.mutable_input_stream()},
{node.output_stream(), config.mutable_output_stream()},
{node.input_side_packet(), config.mutable_input_side_packet()},
{node.output_side_packet(), config.mutable_output_side_packet()}}) {
for (const auto& conn : item.node_conns) {
*item.graph_conns->Add() = conn;
}
}
*config.add_node() = std::move(node);
return config;
}
} // namespace tool
} // namespace mediapipe
mediapipe/framework/tool/subgraph_expansion.h
+5
View File
@@ -72,6 +72,11 @@ namespace tool {
CalculatorGraphConfig* config,
const GraphRegistry* graph_registry = nullptr);
// Creates a graph wrapping the provided node and exposing all of its
// connections
CalculatorGraphConfig MakeSingleNodeGraph(
CalculatorGraphConfig::Node subgraph_node);
} // namespace tool
} // namespace mediapipe
mediapipe/gpu/BUILD
+1 -1
View File
@@ -942,7 +942,7 @@ objc_library(
ios_unit_test(
name = "gl_ios_test",
minimum_os_version = MIN_IOS_VERSION,
runner = "//googlemac/iPhone/Shared/Testing/EarlGrey/Runner:IOS_LATEST",
runner = "//testing/utp/ios:IOS_LATEST",
tags = [
"ios",
],
mediapipe/gpu/gl_simple_shaders.cc
+1
View File
@@ -62,6 +62,7 @@ namespace mediapipe {
PRECISION_COMPAT \
"#if __VERSION__ < 130\n" \
"#define in varying\n" \
"#define texture texture2D\n" \
"#if defined(GL_ES) && !defined(GL_FRAGMENT_PRECISION_HIGH)\n" \
"#define highp mediump\n" \
"#endif // GL_ES && !GL_FRAGMENT_PRECISION_HIGH\n" \
mediapipe/graphs/hand_tracking/BUILD
+7
View File
@@ -102,6 +102,13 @@ cc_library(
],
)
mediapipe_binary_graph(
name = "multi_hand_tracking_desktop_live_binary_graph",
graph = "multi_hand_tracking_desktop_live.pbtxt",
output_name = "multi_hand_tracking_desktop_live.binarypb",
deps = [":multi_hand_desktop_tflite_calculators"],
)
mediapipe_binary_graph(
name = "multi_hand_tracking_mobile_gpu_binary_graph",
graph = "multi_hand_tracking_mobile.pbtxt",
mediapipe/graphs/hand_tracking/multi_hand_tracking_desktop_live.pbtxt
+3
View File
@@ -6,6 +6,9 @@
# Images coming into and out of the graph.
input_stream: "input_video"
output_stream: "output_video"
# Palm detections and hand landmarks info.
output_stream: "multi_palm_detections"
output_stream: "multi_hand_landmarks"
# Determines if an input vector of NormalizedRect has a size greater than or
# equal to the provided min_size.
mediapipe/graphs/instant_motion_tracking/calculators/BUILD
-2
View File
@@ -18,8 +18,6 @@ licenses(["notice"])
package(default_visibility = ["//visibility:public"])
exports_files(["LICENSE"])
proto_library(
name = "sticker_buffer_proto",
srcs = [
mediapipe/graphs/object_detection_3d/calculators/gl_animation_overlay_calculator.cc
+13 -4
View File
@@ -77,10 +77,19 @@ static const float kModelMatrix[] = {0.83704215, -0.36174262, 0.41049102, 0.0,
// Texture to use with animation file. Texture is REQUIRED to be passed into
// the calculator, but can be passed in as a Side Packet OR Input Stream.
// ANIMATION_ASSET (String, required):
// Path of animation file to load and render. Should be generated by
// //java/com/google/android/apps/motionstills/SimpleObjEncryptor with
// --compressed_mode=true. See comments and documentation there for more
// information on custom .obj.uuu file format.
// Path of animation file to load and render. The file format expects an
// arbitrary number of animation frames, concatenated directly together,
// with each animation frame looking like:
// HEADER
// VERTICES
// TEXTURE_COORDS
// INDICES
// The header consists of 3 int32 lengths, the sizes of the vertex data,
// the texcoord data, and the index data, respectively. Let us call those
// N1, N2, and N3. Then we expect N1 float32's for vertex information
// (x1,y1,z1,x2,y2,z2,etc.), followed by N2 float32's for texcoord
// information (u1,v1,u2,v2,u3,v3,etc.), followed by N3 shorts/int16's
// for triangle indices (a1,b1,c1,a2,b2,c2,etc.).
// CAMERA_PARAMETERS_PROTO_STRING (String, optional):
// Serialized proto std::string of CameraParametersProto. We need this to
// get the right aspect ratio and field of view.
mediapipe/java/com/google/mediapipe/components/BUILD
+5 -1
View File
@@ -47,14 +47,18 @@ android_library(
],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/java/com/google/mediapipe/glutil",
"//third_party:androidx_appcompat",
"//third_party:androidx_core",
"//third_party:androidx_legacy_support_v4",
"//third_party:camera2",
"//third_party:camerax_camera2",
"//third_party:camerax_core",
"//third_party:camerax_lifecycle",
"@maven//:androidx_concurrent_concurrent_futures",
"@maven//:androidx_lifecycle_lifecycle_common",
"@maven//:com_google_code_findbugs_jsr305",
"@maven//:com_google_guava_guava",
"@maven//:com_google_guava_listenablefuture",
],
)
mediapipe/java/com/google/mediapipe/components/CameraXPreviewHelper.java
+204 -48
View File
@@ -23,16 +23,28 @@ import android.hardware.camera2.CameraCharacteristics;
import android.hardware.camera2.CameraManager;
import android.hardware.camera2.CameraMetadata;
import android.hardware.camera2.params.StreamConfigurationMap;
import android.opengl.GLES20;
import android.os.Handler;
import android.os.HandlerThread;
import android.os.Process;
import android.os.SystemClock;
import android.util.Log;
import android.util.Size;
import android.view.Surface;
import androidx.camera.core.Camera;
import androidx.camera.core.CameraSelector;
import androidx.camera.core.CameraX;
import androidx.camera.core.CameraX.LensFacing;
import androidx.camera.core.Preview;
import androidx.camera.core.PreviewConfig;
import androidx.camera.lifecycle.ProcessCameraProvider;
import androidx.core.content.ContextCompat;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.mediapipe.glutil.EglManager;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.Executor;
import java.util.concurrent.RejectedExecutionException;
import javax.annotation.Nullable;
import javax.microedition.khronos.egl.EGLSurface;
/**
* Uses CameraX APIs for camera setup and access.
@@ -40,6 +52,43 @@ import javax.annotation.Nullable;
* <p>{@link CameraX} connects to the camera and provides video frames.
*/
public class CameraXPreviewHelper extends CameraHelper {
/**
* Provides an Executor that wraps a single-threaded Handler.
*
* <p>All operations involving the surface texture should happen in a single thread, and that
* thread should not be the main thread.
*
* <p>The surface provider callbacks require an Executor, and the onFrameAvailable callback
* requires a Handler. We want everything to run on the same thread, so we need an Executor that
* is also a Handler.
*/
private static final class SingleThreadHandlerExecutor implements Executor {
private final HandlerThread handlerThread;
private final Handler handler;
SingleThreadHandlerExecutor(String threadName, int priority) {
handlerThread = new HandlerThread(threadName, priority);
handlerThread.start();
handler = new Handler(handlerThread.getLooper());
}
Handler getHandler() {
return handler;
}
@Override
public void execute(Runnable command) {
if (!handler.post(command)) {
throw new RejectedExecutionException(handlerThread.getName() + " is shutting down.");
}
}
boolean shutdown() {
return handlerThread.quitSafely();
}
}
private static final String TAG = "CameraXPreviewHelper";
// Target frame and view resolution size in landscape.
@@ -48,7 +97,12 @@ public class CameraXPreviewHelper extends CameraHelper {
// Number of attempts for calculating the offset between the camera's clock and MONOTONIC clock.
private static final int CLOCK_OFFSET_CALIBRATION_ATTEMPTS = 3;
private final SingleThreadHandlerExecutor renderExecutor =
new SingleThreadHandlerExecutor("RenderThread", Process.THREAD_PRIORITY_DEFAULT);
private ProcessCameraProvider cameraProvider;
private Preview preview;
private Camera camera;
// Size of the camera-preview frames from the camera.
private Size frameSize;
@@ -73,51 +127,93 @@ public class CameraXPreviewHelper extends CameraHelper {
}
public void startCamera(
Activity context, CameraFacing cameraFacing, SurfaceTexture surfaceTexture, Size targetSize) {
if (targetSize == null) {
targetSize = TARGET_SIZE;
}
Activity context,
CameraFacing cameraFacing,
SurfaceTexture unusedSurfaceTexture,
Size targetSize) {
Executor mainThreadExecutor = ContextCompat.getMainExecutor(context);
ListenableFuture<ProcessCameraProvider> cameraProviderFuture =
ProcessCameraProvider.getInstance(context);
LensFacing cameraLensFacing =
cameraFacing == CameraHelper.CameraFacing.FRONT ? LensFacing.FRONT : LensFacing.BACK;
PreviewConfig previewConfig =
new PreviewConfig.Builder()
.setLensFacing(cameraLensFacing)
.setTargetResolution(targetSize)
.build();
preview = new Preview(previewConfig);
targetSize = (targetSize == null ? TARGET_SIZE : targetSize);
// According to CameraX documentation
// (https://developer.android.com/training/camerax/configuration#specify-resolution):
// "Express the resolution Size in the coordinate frame after rotating the supported sizes by
// the target rotation."
// Since we only support portrait orientation, we unconditionally transpose width and height.
Size rotatedSize =
new Size(/* width= */ targetSize.getHeight(), /* height= */ targetSize.getWidth());
preview.setOnPreviewOutputUpdateListener(
previewOutput -> {
if (!previewOutput.getTextureSize().equals(frameSize)) {
frameSize = previewOutput.getTextureSize();
frameRotation = previewOutput.getRotationDegrees();
if (frameSize.getWidth() == 0 || frameSize.getHeight() == 0) {
// Invalid frame size. Wait for valid input dimensions before updating display size.
Log.d(TAG, "Invalid frameSize.");
return;
cameraProviderFuture.addListener(
() -> {
try {
cameraProvider = cameraProviderFuture.get();
} catch (Exception e) {
if (e instanceof InterruptedException) {
Thread.currentThread().interrupt();
}
Log.e(TAG, "Unable to get ProcessCameraProvider: ", e);
return;
}
Integer selectedLensFacing =
preview = new Preview.Builder().setTargetResolution(rotatedSize).build();
CameraSelector cameraSelector =
cameraFacing == CameraHelper.CameraFacing.FRONT
? CameraMetadata.LENS_FACING_FRONT
: CameraMetadata.LENS_FACING_BACK;
cameraCharacteristics = getCameraCharacteristics(context, selectedLensFacing);
if (cameraCharacteristics != null) {
// Queries camera timestamp source. It should be one of REALTIME or UNKNOWN as
// documented in
// https://developer.android.com/reference/android/hardware/camera2/CameraCharacteristics.html#SENSOR_INFO_TIMESTAMP_SOURCE.
cameraTimestampSource =
cameraCharacteristics.get(CameraCharacteristics.SENSOR_INFO_TIMESTAMP_SOURCE);
focalLengthPixels = calculateFocalLengthInPixels();
}
? CameraSelector.DEFAULT_FRONT_CAMERA
: CameraSelector.DEFAULT_BACK_CAMERA;
if (onCameraStartedListener != null) {
onCameraStartedListener.onCameraStarted(previewOutput.getSurfaceTexture());
}
});
CameraX.bindToLifecycle(/*lifecycleOwner=*/ (LifecycleOwner) context, preview);
// Provide surface texture.
preview.setSurfaceProvider(
renderExecutor,
request -> {
Size resolution = request.getResolution();
Log.d(
TAG,
String.format(
"Received surface request for resolution %dx%d",
resolution.getWidth(), resolution.getHeight()));
SurfaceTexture previewFrameTexture = createSurfaceTexture();
previewFrameTexture.setDefaultBufferSize(
resolution.getWidth(), resolution.getHeight());
previewFrameTexture.setOnFrameAvailableListener(
frameTexture -> {
if (frameTexture != previewFrameTexture) {
return;
}
onInitialFrameReceived(context, frameTexture);
},
renderExecutor.getHandler());
Surface surface = new Surface(previewFrameTexture);
Log.d(TAG, "Providing surface");
request.provideSurface(
surface,
renderExecutor,
result -> {
Log.d(TAG, "Surface request result: " + result);
// Per
// https://developer.android.com/reference/androidx/camera/core/SurfaceRequest.Result,
// the surface was either never used (RESULT_INVALID_SURFACE,
// RESULT_REQUEST_CANCELLED, RESULT_SURFACE_ALREADY_PROVIDED) or the surface
// was used successfully and was eventually detached
// (RESULT_SURFACE_USED_SUCCESSFULLY) so we can release it now to free up
// resources.
previewFrameTexture.release();
surface.release();
});
});
// If we pause/resume the activity, we need to unbind the earlier preview use case, given
// the way the activity is currently structured.
cameraProvider.unbindAll();
// Bind preview use case to camera.
camera =
cameraProvider.bindToLifecycle(
/*lifecycleOwner=*/ (LifecycleOwner) context, cameraSelector, preview);
},
mainThreadExecutor);
}
@Override
@@ -127,17 +223,16 @@ public class CameraXPreviewHelper extends CameraHelper {
@Override
public Size computeDisplaySizeFromViewSize(Size viewSize) {
if (viewSize == null || frameSize == null) {
// Wait for all inputs before setting display size.
Log.d(TAG, "viewSize or frameSize is null.");
return null;
}
Size optimalSize = getOptimalViewSize(viewSize);
return optimalSize != null ? optimalSize : frameSize;
// Camera target size is computed already, so just return the capture frame size.
return frameSize;
}
@Nullable
// TODO: Compute optimal view size from available stream sizes.
// Currently, we create the preview stream before we know what size our preview SurfaceView is.
// Instead, we should determine our optimal stream size (based on resolution and aspect ratio
// difference with the preview SurfaceView) and open the preview stream then. Until we make that
// change, this method is unused.
private Size getOptimalViewSize(Size targetSize) {
if (cameraCharacteristics != null) {
StreamConfigurationMap map =
@@ -221,6 +316,56 @@ public class CameraXPreviewHelper extends CameraHelper {
return frameSize;
}
private void onInitialFrameReceived(Activity context, SurfaceTexture previewFrameTexture) {
// This method is called by the onFrameAvailableListener we install when opening the camera
// session, the first time we receive a frame. In this method, we remove our callback,
// acknowledge the frame (via updateTextImage()), detach the texture from the GL context we
// created earlier (so that the MediaPipe pipeline can attach it), and perform some other
// one-time initialization based on the newly opened camera device. Finally, we indicate the
// camera session is ready via the onCameraStartedListener.
// Remove our callback.
previewFrameTexture.setOnFrameAvailableListener(null);
// Update texture image so we don't stall callbacks.
previewFrameTexture.updateTexImage();
// Detach the SurfaceTexture from the GL context we created earlier so that the MediaPipe
// pipeline can attach it.
previewFrameTexture.detachFromGLContext();
if (!preview.getAttachedSurfaceResolution().equals(frameSize)) {
frameSize = preview.getAttachedSurfaceResolution();
frameRotation = camera.getCameraInfo().getSensorRotationDegrees();
if (frameSize.getWidth() == 0 || frameSize.getHeight() == 0) {
// Invalid frame size. Wait for valid input dimensions before updating
// display size.
Log.d(TAG, "Invalid frameSize.");
return;
}
}
Integer selectedLensFacing =
cameraFacing == CameraHelper.CameraFacing.FRONT
? CameraMetadata.LENS_FACING_FRONT
: CameraMetadata.LENS_FACING_BACK;
cameraCharacteristics = getCameraCharacteristics(context, selectedLensFacing);
if (cameraCharacteristics != null) {
// Queries camera timestamp source. It should be one of REALTIME or UNKNOWN
// as documented in
// https://developer.android.com/reference/android/hardware/camera2/CameraCharacteristics.html#SENSOR_INFO_TIMESTAMP_SOURCE.
cameraTimestampSource =
cameraCharacteristics.get(CameraCharacteristics.SENSOR_INFO_TIMESTAMP_SOURCE);
focalLengthPixels = calculateFocalLengthInPixels();
}
OnCameraStartedListener listener = onCameraStartedListener;
if (listener != null) {
ContextCompat.getMainExecutor(context)
.execute(() -> listener.onCameraStarted(previewFrameTexture));
}
}
// Computes the focal length of the camera in pixels based on lens and sensor properties.
private float calculateFocalLengthInPixels() {
// Focal length of the camera in millimeters.
@@ -237,6 +382,17 @@ public class CameraXPreviewHelper extends CameraHelper {
return frameSize.getWidth() * focalLengthMm / sensorWidthMm;
}
private static SurfaceTexture createSurfaceTexture() {
// Create a temporary surface to make the context current.
EglManager eglManager = new EglManager(null);
EGLSurface tempEglSurface = eglManager.createOffscreenSurface(1, 1);
eglManager.makeCurrent(tempEglSurface, tempEglSurface);
int[] textures = new int[1];
GLES20.glGenTextures(1, textures, 0);
SurfaceTexture previewFrameTexture = new SurfaceTexture(textures[0]);
return previewFrameTexture;
}
@Nullable
private static CameraCharacteristics getCameraCharacteristics(
Activity context, Integer lensFacing) {
mediapipe/modules/face_landmark/face_landmark_landmarks_to_roi.pbtxt
+1 -1
View File
@@ -31,7 +31,7 @@ node {
options: {
[mediapipe.DetectionsToRectsCalculatorOptions.ext] {
rotation_vector_start_keypoint_index: 33 # Left side of left eye.
rotation_vector_end_keypoint_index: 133 # Right side of right eye.
rotation_vector_end_keypoint_index: 263 # Right side of right eye.
rotation_vector_target_angle_degrees: 0
}
}
mediapipe/objc/BUILD
+16 -5
View File
@@ -90,16 +90,27 @@ objc_library(
objc_library(
name = "mediapipe_input_sources_ios",
srcs = [
"MPPCameraInputSource.m",
"MPPDisplayLinkWeakTarget.m",
"MPPInputSource.m",
"MPPPlayerInputSource.m",
],
] + select({
"//mediapipe:ios": [
"MPPCameraInputSource.m",
"MPPDisplayLinkWeakTarget.m",
],
"//conditions:default": [],
}),
hdrs = [
"MPPCameraInputSource.h",
"MPPDisplayLinkWeakTarget.h",
"MPPInputSource.h",
"MPPPlayerInputSource.h",
] + select({
"//mediapipe:ios": [
"MPPCameraInputSource.h",
"MPPDisplayLinkWeakTarget.h",
],
"//conditions:default": [],
}),
sdk_frameworks = [
"CoreVideo",
],
visibility = ["//mediapipe/framework:mediapipe_internal"],
)
mediapipe/objc/MPPPlayerInputSource.m
+43 -1
View File
@@ -12,16 +12,24 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#import <CoreVideo/CoreVideo.h>
#import "MPPPlayerInputSource.h"
#if !TARGET_OS_OSX
#import "mediapipe/objc/MPPDisplayLinkWeakTarget.h"
#endif
@implementation MPPPlayerInputSource {
AVAsset* _video;
AVPlayerItem* _videoItem;
AVPlayer* _videoPlayer;
AVPlayerItemVideoOutput* _videoOutput;
#if !TARGET_OS_OSX
CADisplayLink* _videoDisplayLink;
MPPDisplayLinkWeakTarget* _displayLinkWeakTarget;
#else
CVDisplayLinkRef _videoDisplayLink;
#endif // TARGET_OS_OSX
id _videoEndObserver;
}
@@ -40,6 +48,7 @@
_videoOutput.suppressesPlayerRendering = YES;
[_videoItem addOutput:_videoOutput];
#if !TARGET_OS_OSX
_displayLinkWeakTarget =
[[MPPDisplayLinkWeakTarget alloc] initWithTarget:self selector:@selector(videoUpdate:)];
@@ -47,7 +56,15 @@
selector:@selector(displayLinkCallback:)];
_videoDisplayLink.paused = YES;
[_videoDisplayLink addToRunLoop:[NSRunLoop mainRunLoop] forMode:NSRunLoopCommonModes];
#else
CGDirectDisplayID displayID = CGMainDisplayID();
CVReturn error = CVDisplayLinkCreateWithCGDisplay(displayID, &_videoDisplayLink);
if (error) {
_videoDisplayLink = NULL;
}
CVDisplayLinkStop(_videoDisplayLink);
CVDisplayLinkSetOutputCallback(_videoDisplayLink, renderCallback, (__bridge void*)self);
#endif // TARGET_OS_OSX
_videoPlayer = [AVPlayer playerWithPlayerItem:_videoItem];
_videoPlayer.actionAtItemEnd = AVPlayerActionAtItemEndNone;
NSNotificationCenter* center = [NSNotificationCenter defaultCenter];
@@ -65,11 +82,19 @@
- (void)start {
[_videoPlayer play];
#if !TARGET_OS_OSX
_videoDisplayLink.paused = NO;
#else
CVDisplayLinkStart(_videoDisplayLink);
#endif
}
- (void)stop {
#if !TARGET_OS_OSX
_videoDisplayLink.paused = YES;
#else
CVDisplayLinkStop(_videoDisplayLink);
#endif
[_videoPlayer pause];
}
@@ -77,7 +102,20 @@
return _videoPlayer.rate != 0.0;
}
#if !TARGET_OS_OSX
- (void)videoUpdate:(CADisplayLink*)sender {
[self videoUpdateIfNeeded];
}
#else
static CVReturn renderCallback(CVDisplayLinkRef displayLink, const CVTimeStamp* inNow,
const CVTimeStamp* inOutputTime, CVOptionFlags flagsIn,
CVOptionFlags* flagsOut, void* displayLinkContext) {
[(__bridge MPPPlayerInputSource*)displayLinkContext videoUpdateIfNeeded];
return kCVReturnSuccess;
}
#endif // TARGET_OS_OSX
- (void)videoUpdateIfNeeded {
CMTime timestamp = [_videoItem currentTime];
if ([_videoOutput hasNewPixelBufferForItemTime:timestamp]) {
CVPixelBufferRef pixelBuffer =
@@ -96,7 +134,11 @@
- (void)dealloc {
[[NSNotificationCenter defaultCenter] removeObserver:self];
#if !TARGET_OS_OSX
[_videoDisplayLink invalidate];
#else
CVDisplayLinkRelease(_videoDisplayLink);
#endif
_videoPlayer = nil;
}
mediapipe/python/pybind/BUILD
+1
View File
@@ -43,6 +43,7 @@ pybind_library(
deps = [
":image_frame_util",
":util",
"//mediapipe/framework:type_map",
],
)
mediapipe/python/pybind/calculator_graph.cc
+8 -6
View File
@@ -98,12 +98,14 @@ void CalculatorGraphSubmodule(pybind11::module* module) {
if ((init_with_binary_graph ? 1 : 0) + (init_with_graph_proto ? 1 : 0) +
(init_with_validated_graph_config ? 1 : 0) !=
1) {
throw RaisePyError(
PyExc_ValueError,
"Please provide \'binary_graph\' to initialize the graph with"
" binary graph or provide \'graph_config\' to initialize the "
" with graph config proto or provide \'validated_graph_config\' "
" to initialize the with ValidatedGraphConfig object.");
throw RaisePyError(PyExc_ValueError,
"Please provide one of the following: "
"\'binary_graph_path\' to initialize the graph "
"with a binary graph file, or "
"\'graph_config\' to initialize the graph with a "
"graph config proto, or "
"\'validated_graph_config\' to initialize the "
"graph with a ValidatedGraphConfig object.");
}
auto calculator_graph = absl::make_unique<CalculatorGraph>();
RaisePyErrorIfNotOk(calculator_graph->Initialize(graph_config_proto));
mediapipe/python/pybind/image_frame.cc
+4
View File
@@ -365,3 +365,7 @@ void ImageFrameSubmodule(pybind11::module* module) {
} // namespace python
} // namespace mediapipe
#include "mediapipe/framework/type_map.h"
MEDIAPIPE_REGISTER_TYPE(::mediapipe::ImageFrame, "::mediapipe::ImageFrame",
nullptr, nullptr);
mediapipe/python/pybind/validated_graph_config.cc
+8
View File
@@ -65,6 +65,14 @@ void ValidatedGraphConfigSubmodule(pybind11::module* module) {
.c_str());
}
}
if (!(init_with_binary_graph ^ init_with_graph_proto)) {
throw RaisePyError(
PyExc_ValueError,
"Please either provide \'binary_graph_path\' to initialize "
"a ValidatedGraphConfig object with a binary graph file or "
"\'graph_config\' to initialize a ValidatedGraphConfig "
"object with a graph config proto.");
}
RaisePyErrorIfNotOk(self->Initialize(graph_config_proto));
},
R"doc(Initialize ValidatedGraphConfig with a CalculatorGraphConfig.
mediapipe/util/annotation_renderer.h
+1
View File
@@ -76,6 +76,7 @@ class AnnotationRenderer {
// Should be in the range (0-1].
// See 'gpu_scale_factor' in annotation_overlay_calculator.proto
void SetScaleFactor(float scale_factor);
float GetScaleFactor() { return scale_factor_; }
private:
// Draws a rectangle on the image as described in the annotation.
mediapipe/util/sequence/media_sequence.py
+11 -4
View File
@@ -407,10 +407,17 @@ def _create_region_with_prefix(name, prefix):
get_bbox_xmax_at(index, sequence_example, prefix=prefix)),
1)
def add_prefixed_bbox(values, sequence_example, prefix):
add_bbox_ymin(values[:, 0], sequence_example, prefix=prefix)
add_bbox_xmin(values[:, 1], sequence_example, prefix=prefix)
add_bbox_ymax(values[:, 2], sequence_example, prefix=prefix)
add_bbox_xmax(values[:, 3], sequence_example, prefix=prefix)
values = np.array(values)
if values.size == 0:
add_bbox_ymin([], sequence_example, prefix=prefix)
add_bbox_xmin([], sequence_example, prefix=prefix)
add_bbox_ymax([], sequence_example, prefix=prefix)
add_bbox_xmax([], sequence_example, prefix=prefix)
else:
add_bbox_ymin(values[:, 0], sequence_example, prefix=prefix)
add_bbox_xmin(values[:, 1], sequence_example, prefix=prefix)
add_bbox_ymax(values[:, 2], sequence_example, prefix=prefix)
add_bbox_xmax(values[:, 3], sequence_example, prefix=prefix)
def get_prefixed_bbox_size(sequence_example, prefix):
return get_bbox_ymin_size(sequence_example, prefix=prefix)
def has_prefixed_bbox(sequence_example, prefix):
mediapipe/util/sequence/media_sequence_test.py
+2 -1
View File
@@ -128,8 +128,9 @@ class MediaSequenceTest(tf.test.TestCase):
example = tf.train.SequenceExample()
boxes = np.array([[0.1, 0.2, 0.3, 0.4],
[0.5, 0.6, 0.7, 0.8]])
empty_boxes = np.array([])
ms.add_bbox(boxes, example)
ms.add_bbox(boxes, example)
ms.add_bbox(empty_boxes, example)
self.assertEqual(2, ms.get_bbox_size(example))
self.assertAllClose(boxes, ms.get_bbox_at(0, example))
self.assertTrue(ms.has_bbox(example))
mediapipe/util/tflite/BUILD
+7
View File
@@ -32,13 +32,20 @@ cc_library(
name = "cpu_op_resolver",
srcs = ["cpu_op_resolver.cc"],
hdrs = ["cpu_op_resolver.h"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/port:logging",
"//mediapipe/util/tflite/operations:max_pool_argmax",
"//mediapipe/util/tflite/operations:max_unpooling",
"//mediapipe/util/tflite/operations:transpose_conv_bias",
"@org_tensorflow//tensorflow/lite:builtin_op_data",
"@org_tensorflow//tensorflow/lite:framework",
"@org_tensorflow//tensorflow/lite/kernels:builtin_ops",
],
# For using the symbol `MediaPipe_RegisterTfLiteOpResolver` in Python
# with `tensorflow.lite.python.interpreter.InterpreterWithCustomOps`.
alwayslink = 1,
)
cc_library(
mediapipe/util/tflite/cpu_op_resolver.cc
+10 -6
View File
@@ -14,19 +14,23 @@
#include "mediapipe/util/tflite/cpu_op_resolver.h"
#include "mediapipe/framework/port/logging.h"
#include "mediapipe/util/tflite/operations/max_pool_argmax.h"
#include "mediapipe/util/tflite/operations/max_unpooling.h"
#include "mediapipe/util/tflite/operations/transpose_conv_bias.h"
#include "tensorflow/lite/builtin_op_data.h"
#include "tensorflow/lite/mutable_op_resolver.h"
namespace mediapipe {
CpuOpResolver::CpuOpResolver() {
AddCustom("MaxPoolingWithArgmax2D",
tflite_operations::RegisterMaxPoolingWithArgmax2D());
AddCustom("MaxUnpooling2D", tflite_operations::RegisterMaxUnpooling2D());
AddCustom("Convolution2DTransposeBias",
tflite_operations::RegisterConvolution2DTransposeBias());
void MediaPipe_RegisterTfLiteOpResolver(tflite::MutableOpResolver *resolver) {
CHECK(resolver != nullptr);
resolver->AddCustom("MaxPoolingWithArgmax2D",
tflite_operations::RegisterMaxPoolingWithArgmax2D());
resolver->AddCustom("MaxUnpooling2D",
tflite_operations::RegisterMaxUnpooling2D());
resolver->AddCustom("Convolution2DTransposeBias",
tflite_operations::RegisterConvolution2DTransposeBias());
}
} // namespace mediapipe
mediapipe/util/tflite/cpu_op_resolver.h
+6 -2
View File
@@ -19,13 +19,17 @@
namespace mediapipe {
// This OpResolver is used for supporting the following ops on CPU.:
// This function registers the CPU implementations for following custom ops:
// "Convolution2DTransposeBias"
// "MaxPoolArgmax"
// "MaxUnpooling"
extern "C" void MediaPipe_RegisterTfLiteOpResolver(tflite::MutableOpResolver*);
// This resolver is used for the custom ops introduced by
// `MediaPipe_RegisterTfLiteOpResolver` (see above).
class CpuOpResolver : public tflite::ops::builtin::BuiltinOpResolver {
public:
CpuOpResolver();
CpuOpResolver() { MediaPipe_RegisterTfLiteOpResolver(this); }
};
} // namespace mediapipe
mediapipe/util/tflite/tflite_gpu_runner.cc
+12 -5
View File
@@ -100,12 +100,19 @@ mediapipe::Status TFLiteGPURunner::Build() {
// 1. Prepare inference builder.
std::unique_ptr<InferenceBuilder> builder;
// By default, we try CL first & fall back to GL if that fails.
absl::Status status = InitializeOpenCL(&builder);
if (status.ok()) {
LOG(INFO) << "OpenCL backend is used.";
} else {
LOG(ERROR) << "Falling back to OpenGL: " << status.message();
if (opencl_is_forced_) {
MP_RETURN_IF_ERROR(InitializeOpenCL(&builder));
} else if (opengl_is_forced_) {
MP_RETURN_IF_ERROR(InitializeOpenGL(&builder));
} else {
// try to build OpenCL first. If something goes wrong, fall back to OpenGL.
absl::Status status = InitializeOpenCL(&builder);
if (status.ok()) {
LOG(INFO) << "OpenCL backend is used.";
} else {
LOG(ERROR) << "Falling back to OpenGL: " << status.message();
MP_RETURN_IF_ERROR(InitializeOpenGL(&builder));
}
}
// Both graphs are not needed anymore. Make sure they are deleted.
mediapipe/util/tflite/tflite_gpu_runner.h
+7
View File
@@ -56,6 +56,10 @@ class TFLiteGPURunner {
mediapipe::Status InitializeWithModel(
const tflite::FlatBufferModel& flatbuffer,
const tflite::OpResolver& op_resolver);
void ForceOpenGL() { opengl_is_forced_ = true; }
void ForceOpenCL() { opencl_is_forced_ = true; }
mediapipe::Status BindSSBOToInputTensor(GLuint ssbo_id, int input_id);
mediapipe::Status BindSSBOToOutputTensor(GLuint ssbo_id, int output_id);
@@ -105,6 +109,9 @@ class TFLiteGPURunner {
// after graph_ becomes "converted" into runner_.
std::vector<BHWC> input_shapes_;
std::vector<BHWC> output_shapes_;
bool opencl_is_forced_ = false;
bool opengl_is_forced_ = false;
};
} // namespace gpu
mediapipe/util/tracking/tracking.cc
+4 -3
View File
@@ -1945,9 +1945,10 @@ void MotionBox::EstimateObjectMotion(
// For any additional degrees of freedom, require a good set of inliers.
if (num_continued_inliers < options_.object_similarity_min_contd_inliers()) {
VLOG_IF(2, options_.tracking_degrees() !=
TrackStepOptions::TRACKING_DEGREE_TRANSLATION)
<< "Falling back to translation!!!";
if (options_.tracking_degrees() !=
TrackStepOptions::TRACKING_DEGREE_TRANSLATION) {
VLOG(2) << "Falling back to translation!!!";
}
VLOG(1) << "num_continued_inliers: " << num_continued_inliers << " < "
<< options_.object_similarity_min_contd_inliers()
<< ", fall back to translation";